1 // SPDX-License-Identifier: GPL-2.0
3 * NVMe Over Fabrics Target Passthrough command implementation.
5 * Copyright (c) 2017-2018 Western Digital Corporation or its
7 * Copyright (c) 2019-2020, Eideticom Inc.
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 #include <linux/module.h>
13 #include "../host/nvme.h"
16 MODULE_IMPORT_NS(NVME_TARGET_PASSTHRU);
19 * xarray to maintain one passthru subsystem per nvme controller.
21 static DEFINE_XARRAY(passthru_subsystems);
23 static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req)
25 struct nvmet_ctrl *ctrl = req->sq->ctrl;
26 struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl;
27 u16 status = NVME_SC_SUCCESS;
28 struct nvme_id_ctrl *id;
32 id = kzalloc(sizeof(*id), GFP_KERNEL);
34 return NVME_SC_INTERNAL;
36 status = nvmet_copy_from_sgl(req, 0, id, sizeof(*id));
40 id->cntlid = cpu_to_le16(ctrl->cntlid);
41 id->ver = cpu_to_le32(ctrl->subsys->ver);
44 * The passthru NVMe driver may have a limit on the number of segments
45 * which depends on the host's memory fragementation. To solve this,
46 * ensure mdts is limited to the pages equal to the number of segments.
48 max_hw_sectors = min_not_zero(pctrl->max_segments << (PAGE_SHIFT - 9),
49 pctrl->max_hw_sectors);
51 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
53 id->mdts = ilog2(max_hw_sectors) + 9 - page_shift;
57 * We export aerl limit for the fabrics controller, update this when
58 * passthru based aerl support is added.
60 id->aerl = NVMET_ASYNC_EVENTS - 1;
62 /* emulate kas as most of the PCIe ctrl don't have a support for kas */
63 id->kas = cpu_to_le16(NVMET_KAS);
65 /* don't support host memory buffer */
69 id->sqes = min_t(__u8, ((0x6 << 4) | 0x6), id->sqes);
70 id->cqes = min_t(__u8, ((0x4 << 4) | 0x4), id->cqes);
71 id->maxcmd = cpu_to_le16(NVMET_MAX_CMD);
73 /* don't support fuse commands */
76 id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */
77 if (ctrl->ops->flags & NVMF_KEYED_SGLS)
78 id->sgls |= cpu_to_le32(1 << 2);
79 if (req->port->inline_data_size)
80 id->sgls |= cpu_to_le32(1 << 20);
83 * When passsthru controller is setup using nvme-loop transport it will
84 * export the passthru ctrl subsysnqn (PCIe NVMe ctrl) and will fail in
85 * the nvme/host/core.c in the nvme_init_subsystem()->nvme_active_ctrl()
86 * code path with duplicate ctr subsynqn. In order to prevent that we
87 * mask the passthru-ctrl subsysnqn with the target ctrl subsysnqn.
89 memcpy(id->subnqn, ctrl->subsysnqn, sizeof(id->subnqn));
91 /* use fabric id-ctrl values */
92 id->ioccsz = cpu_to_le32((sizeof(struct nvme_command) +
93 req->port->inline_data_size) / 16);
94 id->iorcsz = cpu_to_le32(sizeof(struct nvme_completion) / 16);
96 id->msdbd = ctrl->ops->msdbd;
98 /* Support multipath connections with fabrics */
101 /* Disable reservations, see nvmet_parse_passthru_io_cmd() */
102 id->oncs &= cpu_to_le16(~NVME_CTRL_ONCS_RESERVATIONS);
104 status = nvmet_copy_to_sgl(req, 0, id, sizeof(struct nvme_id_ctrl));
111 static u16 nvmet_passthru_override_id_ns(struct nvmet_req *req)
113 u16 status = NVME_SC_SUCCESS;
114 struct nvme_id_ns *id;
117 id = kzalloc(sizeof(*id), GFP_KERNEL);
119 return NVME_SC_INTERNAL;
121 status = nvmet_copy_from_sgl(req, 0, id, sizeof(struct nvme_id_ns));
125 for (i = 0; i < (id->nlbaf + 1); i++)
127 memset(&id->lbaf[i], 0, sizeof(id->lbaf[i]));
129 id->flbas = id->flbas & ~(1 << 4);
132 * Presently the NVMEof target code does not support sending
133 * metadata, so we must disable it here. This should be updated
134 * once target starts supporting metadata.
138 status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));
145 static void nvmet_passthru_execute_cmd_work(struct work_struct *w)
147 struct nvmet_req *req = container_of(w, struct nvmet_req, p.work);
148 struct request *rq = req->p.rq;
151 nvme_execute_passthru_rq(rq);
153 status = nvme_req(rq)->status;
154 if (status == NVME_SC_SUCCESS &&
155 req->cmd->common.opcode == nvme_admin_identify) {
156 switch (req->cmd->identify.cns) {
157 case NVME_ID_CNS_CTRL:
158 nvmet_passthru_override_id_ctrl(req);
161 nvmet_passthru_override_id_ns(req);
166 req->cqe->result = nvme_req(rq)->result;
167 nvmet_req_complete(req, status);
168 blk_mq_free_request(rq);
171 static void nvmet_passthru_req_done(struct request *rq,
172 blk_status_t blk_status)
174 struct nvmet_req *req = rq->end_io_data;
176 req->cqe->result = nvme_req(rq)->result;
177 nvmet_req_complete(req, nvme_req(rq)->status);
178 blk_mq_free_request(rq);
181 static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq)
183 int sg_cnt = req->sg_cnt;
184 struct scatterlist *sg;
189 if (req->cmd->common.opcode == nvme_cmd_flush)
191 else if (nvme_is_write(req->cmd))
192 op_flags = REQ_SYNC | REQ_IDLE;
194 bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));
195 bio->bi_end_io = bio_put;
196 bio->bi_opf = req_op(rq) | op_flags;
198 for_each_sg(req->sg, sg, req->sg_cnt, i) {
199 if (bio_add_pc_page(rq->q, bio, sg_page(sg), sg->length,
200 sg->offset) < sg->length) {
207 ret = blk_rq_append_bio(rq, &bio);
216 static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
218 struct nvme_ctrl *ctrl = nvmet_req_passthru_ctrl(req);
219 struct request_queue *q = ctrl->admin_q;
220 struct nvme_ns *ns = NULL;
221 struct request *rq = NULL;
226 if (likely(req->sq->qid != 0)) {
227 u32 nsid = le32_to_cpu(req->cmd->common.nsid);
229 ns = nvme_find_get_ns(ctrl, nsid);
231 pr_err("failed to get passthru ns nsid:%u\n", nsid);
232 status = NVME_SC_INVALID_NS | NVME_SC_DNR;
239 rq = nvme_alloc_request(q, req->cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
241 status = NVME_SC_INTERNAL;
246 ret = nvmet_passthru_map_sg(req, rq);
248 status = NVME_SC_INTERNAL;
254 * If there are effects for the command we are about to execute, or
255 * an end_req function we need to use nvme_execute_passthru_rq()
256 * synchronously in a work item seeing the end_req function and
257 * nvme_passthru_end() can't be called in the request done callback
258 * which is typically in interrupt context.
260 effects = nvme_command_effects(ctrl, ns, req->cmd->common.opcode);
261 if (req->p.use_workqueue || effects) {
262 INIT_WORK(&req->p.work, nvmet_passthru_execute_cmd_work);
264 schedule_work(&req->p.work);
266 rq->end_io_data = req;
267 blk_execute_rq_nowait(rq->q, ns ? ns->disk : NULL, rq, 0,
268 nvmet_passthru_req_done);
277 blk_mq_free_request(rq);
282 nvmet_req_complete(req, status);
286 * We need to emulate set host behaviour to ensure that any requested
287 * behaviour of the target's host matches the requested behaviour
288 * of the device's host and fail otherwise.
290 static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req)
292 struct nvme_ctrl *ctrl = nvmet_req_passthru_ctrl(req);
293 struct nvme_feat_host_behavior *host;
294 u16 status = NVME_SC_INTERNAL;
297 host = kzalloc(sizeof(*host) * 2, GFP_KERNEL);
299 goto out_complete_req;
301 ret = nvme_get_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
302 host, sizeof(*host), NULL);
306 status = nvmet_copy_from_sgl(req, 0, &host[1], sizeof(*host));
310 if (memcmp(&host[0], &host[1], sizeof(host[0]))) {
311 pr_warn("target host has requested different behaviour from the local host\n");
312 status = NVME_SC_INTERNAL;
318 nvmet_req_complete(req, status);
321 static u16 nvmet_setup_passthru_command(struct nvmet_req *req)
323 req->p.use_workqueue = false;
324 req->execute = nvmet_passthru_execute_cmd;
325 return NVME_SC_SUCCESS;
328 u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req)
330 /* Reject any commands with non-sgl flags set (ie. fused commands) */
331 if (req->cmd->common.flags & ~NVME_CMD_SGL_ALL)
332 return NVME_SC_INVALID_FIELD;
334 switch (req->cmd->common.opcode) {
335 case nvme_cmd_resv_register:
336 case nvme_cmd_resv_report:
337 case nvme_cmd_resv_acquire:
338 case nvme_cmd_resv_release:
340 * Reservations cannot be supported properly because the
341 * underlying device has no way of differentiating different
342 * hosts that connect via fabrics. This could potentially be
343 * emulated in the future if regular targets grow support for
346 return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
349 return nvmet_setup_passthru_command(req);
353 * Only features that are emulated or specifically allowed in the list are
354 * passed down to the controller. This function implements the allow list for
355 * both get and set features.
357 static u16 nvmet_passthru_get_set_features(struct nvmet_req *req)
359 switch (le32_to_cpu(req->cmd->features.fid)) {
360 case NVME_FEAT_ARBITRATION:
361 case NVME_FEAT_POWER_MGMT:
362 case NVME_FEAT_LBA_RANGE:
363 case NVME_FEAT_TEMP_THRESH:
364 case NVME_FEAT_ERR_RECOVERY:
365 case NVME_FEAT_VOLATILE_WC:
366 case NVME_FEAT_WRITE_ATOMIC:
367 case NVME_FEAT_AUTO_PST:
368 case NVME_FEAT_TIMESTAMP:
370 case NVME_FEAT_NOPSC:
372 case NVME_FEAT_PLM_CONFIG:
373 case NVME_FEAT_PLM_WINDOW:
374 case NVME_FEAT_HOST_BEHAVIOR:
375 case NVME_FEAT_SANITIZE:
376 case NVME_FEAT_VENDOR_START ... NVME_FEAT_VENDOR_END:
377 return nvmet_setup_passthru_command(req);
379 case NVME_FEAT_ASYNC_EVENT:
380 /* There is no support for forwarding ASYNC events */
381 case NVME_FEAT_IRQ_COALESCE:
382 case NVME_FEAT_IRQ_CONFIG:
383 /* The IRQ settings will not apply to the target controller */
384 case NVME_FEAT_HOST_MEM_BUF:
386 * Any HMB that's set will not be passed through and will
387 * not work as expected
389 case NVME_FEAT_SW_PROGRESS:
391 * The Pre-Boot Software Load Count doesn't make much
392 * sense for a target to export
394 case NVME_FEAT_RESV_MASK:
395 case NVME_FEAT_RESV_PERSIST:
396 /* No reservations, see nvmet_parse_passthru_io_cmd() */
398 return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
402 u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
404 /* Reject any commands with non-sgl flags set (ie. fused commands) */
405 if (req->cmd->common.flags & ~NVME_CMD_SGL_ALL)
406 return NVME_SC_INVALID_FIELD;
409 * Passthru all vendor specific commands
411 if (req->cmd->common.opcode >= nvme_admin_vendor_start)
412 return nvmet_setup_passthru_command(req);
414 switch (req->cmd->common.opcode) {
415 case nvme_admin_async_event:
416 req->execute = nvmet_execute_async_event;
417 return NVME_SC_SUCCESS;
418 case nvme_admin_keep_alive:
420 * Most PCIe ctrls don't support keep alive cmd, we route keep
421 * alive to the non-passthru mode. In future please change this
422 * code when PCIe ctrls with keep alive support available.
424 req->execute = nvmet_execute_keep_alive;
425 return NVME_SC_SUCCESS;
426 case nvme_admin_set_features:
427 switch (le32_to_cpu(req->cmd->features.fid)) {
428 case NVME_FEAT_ASYNC_EVENT:
430 case NVME_FEAT_NUM_QUEUES:
431 case NVME_FEAT_HOST_ID:
432 req->execute = nvmet_execute_set_features;
433 return NVME_SC_SUCCESS;
434 case NVME_FEAT_HOST_BEHAVIOR:
435 req->execute = nvmet_passthru_set_host_behaviour;
436 return NVME_SC_SUCCESS;
438 return nvmet_passthru_get_set_features(req);
441 case nvme_admin_get_features:
442 switch (le32_to_cpu(req->cmd->features.fid)) {
443 case NVME_FEAT_ASYNC_EVENT:
445 case NVME_FEAT_NUM_QUEUES:
446 case NVME_FEAT_HOST_ID:
447 req->execute = nvmet_execute_get_features;
448 return NVME_SC_SUCCESS;
450 return nvmet_passthru_get_set_features(req);
453 case nvme_admin_identify:
454 switch (req->cmd->identify.cns) {
455 case NVME_ID_CNS_CTRL:
456 req->execute = nvmet_passthru_execute_cmd;
457 req->p.use_workqueue = true;
458 return NVME_SC_SUCCESS;
460 req->execute = nvmet_passthru_execute_cmd;
461 req->p.use_workqueue = true;
462 return NVME_SC_SUCCESS;
464 return nvmet_setup_passthru_command(req);
466 case nvme_admin_get_log_page:
467 return nvmet_setup_passthru_command(req);
469 /* Reject commands not in the allowlist above */
470 return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
474 int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
476 struct nvme_ctrl *ctrl;
480 mutex_lock(&subsys->lock);
481 if (!subsys->passthru_ctrl_path)
483 if (subsys->passthru_ctrl)
486 if (subsys->nr_namespaces) {
487 pr_info("cannot enable both passthru and regular namespaces for a single subsystem");
491 ctrl = nvme_ctrl_get_by_path(subsys->passthru_ctrl_path);
494 pr_err("failed to open nvme controller %s\n",
495 subsys->passthru_ctrl_path);
500 old = xa_cmpxchg(&passthru_subsystems, ctrl->cntlid, NULL,
502 if (xa_is_err(old)) {
510 subsys->passthru_ctrl = ctrl;
511 subsys->ver = ctrl->vs;
513 if (subsys->ver < NVME_VS(1, 2, 1)) {
514 pr_warn("nvme controller version is too old: %llu.%llu.%llu, advertising 1.2.1\n",
515 NVME_MAJOR(subsys->ver), NVME_MINOR(subsys->ver),
516 NVME_TERTIARY(subsys->ver));
517 subsys->ver = NVME_VS(1, 2, 1);
520 mutex_unlock(&subsys->lock);
526 mutex_unlock(&subsys->lock);
530 static void __nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
532 if (subsys->passthru_ctrl) {
533 xa_erase(&passthru_subsystems, subsys->passthru_ctrl->cntlid);
534 nvme_put_ctrl(subsys->passthru_ctrl);
536 subsys->passthru_ctrl = NULL;
537 subsys->ver = NVMET_DEFAULT_VS;
540 void nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
542 mutex_lock(&subsys->lock);
543 __nvmet_passthru_ctrl_disable(subsys);
544 mutex_unlock(&subsys->lock);
547 void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
549 mutex_lock(&subsys->lock);
550 __nvmet_passthru_ctrl_disable(subsys);
551 mutex_unlock(&subsys->lock);
552 kfree(subsys->passthru_ctrl_path);