Skip to content

Added vendor specific subsystem pause/resume #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
susobhandey wants to merge 3 commits into
base: v24.01.x-mayastor
Choose a base branch
from
Open

Added vendor specific subsystem pause/resume #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e626f32
Added vendor specific subsystem pause/resume
May 7, 2026
8925ea0
Added rpc option to change the subsystem pause timeout
May 7, 2026
2a710bc
nit: Declare subsystem top of the function
May 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions include/spdk/nvmf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ extern "C" {

#define SPDK_TLS_PSK_MAX_LEN 200

#define SPDK_NVMF_SUBSYSTEM_PAUSE_KEEP_ADMINQ 0x1

struct spdk_nvmf_tgt;
struct spdk_nvmf_subsystem;
struct spdk_nvmf_ctrlr;
Expand Down Expand Up @@ -543,6 +545,21 @@ int spdk_nvmf_subsystem_pause(struct spdk_nvmf_subsystem *subsystem,
spdk_nvmf_subsystem_state_change_done cb_fn,
void *cb_arg);

/**
* Vendor specific wrapper of subsystem pause
* we need to keep allow some admin commands during subsystem pause
*/
int spdk_nvmf_subsystem_pause_ext(struct spdk_nvmf_subsystem *subsystem,
uint32_t nsid,
uint32_t flags,
spdk_nvmf_subsystem_state_change_done cb_fn,
void *cb_arg);

int spdk_nvmf_subsystem_set_pause_timeout(struct spdk_nvmf_subsystem *subsystem,
uint32_t timeout_sec);

uint32_t spdk_nvmf_subsystem_get_pause_timeout(struct spdk_nvmf_subsystem *subsystem);

/**
* Transition an NVMe-oF subsystem from Paused to Active state.
*
Expand All @@ -559,6 +576,13 @@ int spdk_nvmf_subsystem_resume(struct spdk_nvmf_subsystem *subsystem,
spdk_nvmf_subsystem_state_change_done cb_fn,
void *cb_arg);

/**
* Vendor specific wrapper of subsystem resume
* We need to clear vendor specific flags during resume
*/
int spdk_nvmf_subsystem_resume_ext(struct spdk_nvmf_subsystem *subsystem,
spdk_nvmf_subsystem_state_change_done cb_fn,
void *cb_arg);
/**
* Search the target for a subsystem with the given NQN.
*
Expand Down
11 changes: 11 additions & 0 deletions lib/nvmf/ctrlr.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4593,6 +4593,7 @@ nvmf_check_subsystem_active(struct spdk_nvmf_request *req)
struct spdk_nvmf_subsystem_poll_group *sgroup = NULL;
struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
uint32_t nsid;
struct spdk_nvmf_subsystem *subsystem = NULL;
Copy link
Copy Markdown
Member

@tiagolobocastro tiagolobocastro May 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can leave it unset as well, since it's init before being used (also seems to be the practice in spdk AFAIK)


if (spdk_likely(qpair->ctrlr)) {
sgroup = &qpair->group->sgroups[qpair->ctrlr->subsys->id];
Expand All @@ -4606,6 +4607,16 @@ nvmf_check_subsystem_active(struct spdk_nvmf_request *req)
if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC ||
nvmf_qpair_is_admin_queue(qpair))) {
if (sgroup->state != SPDK_NVMF_SUBSYSTEM_ACTIVE) {
subsystem = qpair->ctrlr->subsys;
if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC &&
(subsystem->pause_flags & SPDK_NVMF_SUBSYSTEM_PAUSE_KEEP_ADMINQ)) {
/*
* Vendor maintenance pause
* allow normal admin queue commands while data IO is paused
*/
sgroup->mgmt_io_outstanding++;
return true;
}
/* The subsystem is not currently active. Queue this request. */
TAILQ_INSERT_TAIL(&sgroup->queued, req, link);
return false;
Expand Down
5 changes: 5 additions & 0 deletions lib/nvmf/nvmf_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
/* The spec reserves cntlid values in the range FFF0h to FFFFh. */
#define NVMF_MIN_CNTLID 1
#define NVMF_MAX_CNTLID 0xFFEF
#define SPDK_NVMF_DEFAULT_PAUSE_TIMEOUT_SEC 120

enum spdk_nvmf_tgt_state {
NVMF_TGT_IDLE = 0,
Expand Down Expand Up @@ -320,6 +321,10 @@ struct spdk_nvmf_subsystem {
/* Subsystem event callback and its argument. */
spdk_nvmf_subsystem_event_cb event_cb_fn;
void *event_cb_arg;
/* Vendor specific flags for maintainance work */
uint32_t pause_flags;
struct spdk_poller *pause_timer;
uint32_t pause_timeout_sec;
};

static int
Expand Down
65 changes: 65 additions & 0 deletions lib/nvmf/subsystem.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,9 @@ spdk_nvmf_subsystem_create(struct spdk_nvmf_tgt *tgt,
subsystem->next_cntlid = 0;
subsystem->min_cntlid = NVMF_MIN_CNTLID;
subsystem->max_cntlid = NVMF_MAX_CNTLID;
subsystem->pause_timeout_sec = SPDK_NVMF_DEFAULT_PAUSE_TIMEOUT_SEC;
subsystem->pause_flags = 0;
subsystem->pause_timer = NULL;
snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", nqn);
pthread_mutex_init(&subsystem->mutex, NULL);
TAILQ_INIT(&subsystem->listeners);
Expand Down Expand Up @@ -382,6 +385,11 @@ _nvmf_subsystem_destroy(struct spdk_nvmf_subsystem *subsystem)
return -EINPROGRESS;
}

if (subsystem->pause_timer) {
spdk_poller_unregister(&subsystem->pause_timer);
subsystem->pause_timer = NULL;
}

ns = spdk_nvmf_subsystem_get_first_ns(subsystem);
while (ns != NULL) {
struct spdk_nvmf_ns *next_ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns);
Expand Down Expand Up @@ -758,6 +766,50 @@ spdk_nvmf_subsystem_stop(struct spdk_nvmf_subsystem *subsystem,
return nvmf_subsystem_state_change(subsystem, 0, SPDK_NVMF_SUBSYSTEM_INACTIVE, cb_fn, cb_arg);
}

static
int nvmf_subsys_pause_timer_cb(void *arg)
{
struct spdk_nvmf_subsystem *subsystem = arg;
subsystem->pause_flags = 0;
if (subsystem->pause_timer) {
spdk_poller_unregister(&subsystem->pause_timer);
subsystem->pause_timer = NULL;
}
return SPDK_POLLER_BUSY;
}

int
spdk_nvmf_subsystem_pause_ext(struct spdk_nvmf_subsystem *subsystem,
uint32_t nsid,
uint32_t flags,
spdk_nvmf_subsystem_state_change_done cb_fn,
void *cb_arg)
{
subsystem->pause_flags = flags;
if (subsystem->pause_timer == NULL) {
subsystem->pause_timer = spdk_poller_register(
nvmf_subsys_pause_timer_cb,
subsystem,
subsystem->pause_timeout_sec * 1000000ULL
);
}
return spdk_nvmf_subsystem_pause(subsystem, nsid, cb_fn, cb_arg);
}

int
spdk_nvmf_subsystem_set_pause_timeout(struct spdk_nvmf_subsystem *subsystem,
uint32_t timeout_sec)
{
subsystem->pause_timeout_sec = timeout_sec;
return 0;
}

uint32_t
spdk_nvmf_subsystem_get_pause_timeout(struct spdk_nvmf_subsystem *subsystem)
{
return subsystem->pause_timeout_sec;
}

int
spdk_nvmf_subsystem_pause(struct spdk_nvmf_subsystem *subsystem,
uint32_t nsid,
Expand All @@ -767,6 +819,19 @@ spdk_nvmf_subsystem_pause(struct spdk_nvmf_subsystem *subsystem,
return nvmf_subsystem_state_change(subsystem, nsid, SPDK_NVMF_SUBSYSTEM_PAUSED, cb_fn, cb_arg);
}

int
spdk_nvmf_subsystem_resume_ext(struct spdk_nvmf_subsystem *subsystem,
spdk_nvmf_subsystem_state_change_done cb_fn,
void *cb_arg)
{
subsystem->pause_flags = 0;
if (subsystem->pause_timer) {
spdk_poller_unregister(&subsystem->pause_timer);
subsystem->pause_timer = NULL;
}
return spdk_nvmf_subsystem_resume(subsystem, cb_fn, cb_arg);
}

int
spdk_nvmf_subsystem_resume(struct spdk_nvmf_subsystem *subsystem,
spdk_nvmf_subsystem_state_change_done cb_fn,
Expand Down
50 changes: 50 additions & 0 deletions module/event/subsystems/nvmf/nvmf_rpc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,56 @@ static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_opts_decoder
{"max_subsystems", 0, spdk_json_decode_uint32, true}
};

struct rpc_nvmf_subsystem_set_pause_timeout {
char *nqn;
uint32_t pause_timeout_sec;
};

static const struct spdk_json_object_decoder rpc_nvmf_subsystem_set_pause_timeout_decoders[] = {
{"nqn", offsetof(struct rpc_nvmf_subsystem_set_pause_timeout, nqn), spdk_json_decode_string},
{"pause_timeout_sec", offsetof(struct rpc_nvmf_subsystem_set_pause_timeout, pause_timeout_sec), spdk_json_decode_uint32}
};

static void
rpc_nvmf_subsystem_set_pause_timeout(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_nvmf_subsystem_set_pause_timeout req = {};
struct spdk_nvmf_subsystem *subsystem;
int rc;

if (spdk_json_decode_object(params,
rpc_nvmf_subsystem_set_pause_timeout_decoders,
SPDK_COUNTOF(rpc_nvmf_subsystem_set_pause_timeout_decoders),
&req)) {
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
"Invalid parameters");
goto cleanup;
}

subsystem = spdk_nvmf_tgt_find_subsystem(g_spdk_nvmf_tgt, req.nqn);
if (subsystem == NULL) {
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Invalid nqn");
goto cleanup;
}

rc = spdk_nvmf_subsystem_set_pause_timeout(subsystem, req.pause_timeout_sec);
if (rc != 0) {
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
"Failed to set timeout");
goto cleanup;
}

spdk_jsonrpc_send_bool_response(request, true);

cleanup:
free(req.nqn);
}

SPDK_RPC_REGISTER("nvmf_subsystem_set_pause_timeout",
rpc_nvmf_subsystem_set_pause_timeout,
SPDK_RPC_RUNTIME)

static void
rpc_nvmf_set_max_subsystems(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
Expand Down
12 changes: 12 additions & 0 deletions scripts/rpc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2430,6 +2430,18 @@ def nvmf_set_max_subsystems(args):
p.add_argument('-x', '--max-subsystems', help='Max number of NVMf subsystems', type=int, required=True)
p.set_defaults(func=nvmf_set_max_subsystems)

def nvmf_subsystem_set_pause_timeout(args):
print_json(args.client.nvmf_subsystem_set_pause_timeout(
nqn=args.nqn,
pause_timeout_sec=args.pause_timeout_sec))


p = subparsers.add_parser('nvmf_subsystem_set_pause_timeout',
help='Set NVMf subsystem pause timeout')
p.add_argument('nqn', help='Subsystem NQN')
p.add_argument('pause_timeout_sec', help='Pause timeout in seconds', type=int)
p.set_defaults(func=nvmf_subsystem_set_pause_timeout)

def nvmf_set_config(args):
rpc.nvmf.nvmf_set_config(args.client,
passthru_identify_ctrlr=args.passthru_identify_ctrlr,
Expand Down