2 * NVMe over Fabrics RDMA host code.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/module.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/err.h>
19 #include <linux/string.h>
20 #include <linux/atomic.h>
21 #include <linux/blk-mq.h>
22 #include <linux/blk-mq-rdma.h>
23 #include <linux/types.h>
24 #include <linux/list.h>
25 #include <linux/mutex.h>
26 #include <linux/scatterlist.h>
27 #include <linux/nvme.h>
28 #include <asm/unaligned.h>
30 #include <rdma/ib_verbs.h>
31 #include <rdma/rdma_cm.h>
32 #include <linux/nvme-rdma.h>
38 #define NVME_RDMA_CONNECT_TIMEOUT_MS 3000 /* 3 second */
40 #define NVME_RDMA_MAX_SEGMENTS 256
42 #define NVME_RDMA_MAX_INLINE_SEGMENTS 1
44 struct nvme_rdma_device {
45 struct ib_device *dev;
48 struct list_head entry;
57 struct nvme_rdma_queue;
58 struct nvme_rdma_request {
59 struct nvme_request req;
61 struct nvme_rdma_qe sqe;
62 struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
66 struct ib_reg_wr reg_wr;
67 struct ib_cqe reg_cqe;
68 struct nvme_rdma_queue *queue;
69 struct sg_table sg_table;
70 struct scatterlist first_sgl[];
73 enum nvme_rdma_queue_flags {
74 NVME_RDMA_Q_ALLOCATED = 0,
78 struct nvme_rdma_queue {
79 struct nvme_rdma_qe *rsp_ring;
82 size_t cmnd_capsule_len;
83 struct nvme_rdma_ctrl *ctrl;
84 struct nvme_rdma_device *device;
89 struct rdma_cm_id *cm_id;
91 struct completion cm_done;
94 struct nvme_rdma_ctrl {
95 /* read only in the hot path */
96 struct nvme_rdma_queue *queues;
98 /* other member variables */
99 struct blk_mq_tag_set tag_set;
100 struct work_struct err_work;
102 struct nvme_rdma_qe async_event_sqe;
104 struct delayed_work reconnect_work;
106 struct list_head list;
108 struct blk_mq_tag_set admin_tag_set;
109 struct nvme_rdma_device *device;
113 struct sockaddr_storage addr;
114 struct sockaddr_storage src_addr;
116 struct nvme_ctrl ctrl;
119 static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
121 return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
124 static LIST_HEAD(device_list);
125 static DEFINE_MUTEX(device_list_mutex);
127 static LIST_HEAD(nvme_rdma_ctrl_list);
128 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
131 * Disabling this option makes small I/O goes faster, but is fundamentally
132 * unsafe. With it turned off we will have to register a global rkey that
133 * allows read and write access to all physical memory.
135 static bool register_always = true;
136 module_param(register_always, bool, 0444);
137 MODULE_PARM_DESC(register_always,
138 "Use memory registration even for contiguous memory regions");
140 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
141 struct rdma_cm_event *event);
142 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
144 static const struct blk_mq_ops nvme_rdma_mq_ops;
145 static const struct blk_mq_ops nvme_rdma_admin_mq_ops;
147 /* XXX: really should move to a generic header sooner or later.. */
148 static inline void put_unaligned_le24(u32 val, u8 *p)
155 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
157 return queue - queue->ctrl->queues;
160 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
162 return queue->cmnd_capsule_len - sizeof(struct nvme_command);
165 static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
166 size_t capsule_size, enum dma_data_direction dir)
168 ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
172 static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
173 size_t capsule_size, enum dma_data_direction dir)
175 qe->data = kzalloc(capsule_size, GFP_KERNEL);
179 qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
180 if (ib_dma_mapping_error(ibdev, qe->dma)) {
188 static void nvme_rdma_free_ring(struct ib_device *ibdev,
189 struct nvme_rdma_qe *ring, size_t ib_queue_size,
190 size_t capsule_size, enum dma_data_direction dir)
194 for (i = 0; i < ib_queue_size; i++)
195 nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
199 static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
200 size_t ib_queue_size, size_t capsule_size,
201 enum dma_data_direction dir)
203 struct nvme_rdma_qe *ring;
206 ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
210 for (i = 0; i < ib_queue_size; i++) {
211 if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
218 nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
222 static void nvme_rdma_qp_event(struct ib_event *event, void *context)
224 pr_debug("QP event %s (%d)\n",
225 ib_event_msg(event->event), event->event);
229 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
231 wait_for_completion_interruptible_timeout(&queue->cm_done,
232 msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
233 return queue->cm_error;
236 static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
238 struct nvme_rdma_device *dev = queue->device;
239 struct ib_qp_init_attr init_attr;
242 memset(&init_attr, 0, sizeof(init_attr));
243 init_attr.event_handler = nvme_rdma_qp_event;
245 init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
247 init_attr.cap.max_recv_wr = queue->queue_size + 1;
248 init_attr.cap.max_recv_sge = 1;
249 init_attr.cap.max_send_sge = 1 + NVME_RDMA_MAX_INLINE_SEGMENTS;
250 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
251 init_attr.qp_type = IB_QPT_RC;
252 init_attr.send_cq = queue->ib_cq;
253 init_attr.recv_cq = queue->ib_cq;
255 ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
257 queue->qp = queue->cm_id->qp;
261 static int nvme_rdma_reinit_request(void *data, struct request *rq)
263 struct nvme_rdma_ctrl *ctrl = data;
264 struct nvme_rdma_device *dev = ctrl->device;
265 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
268 if (WARN_ON_ONCE(!req->mr))
271 ib_dereg_mr(req->mr);
273 req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
275 if (IS_ERR(req->mr)) {
276 ret = PTR_ERR(req->mr);
281 req->mr->need_inval = false;
287 static void nvme_rdma_exit_request(struct blk_mq_tag_set *set,
288 struct request *rq, unsigned int hctx_idx)
290 struct nvme_rdma_ctrl *ctrl = set->driver_data;
291 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
292 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
293 struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
294 struct nvme_rdma_device *dev = queue->device;
297 ib_dereg_mr(req->mr);
299 nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
303 static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
304 struct request *rq, unsigned int hctx_idx,
305 unsigned int numa_node)
307 struct nvme_rdma_ctrl *ctrl = set->driver_data;
308 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
309 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
310 struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
311 struct nvme_rdma_device *dev = queue->device;
312 struct ib_device *ibdev = dev->dev;
315 ret = nvme_rdma_alloc_qe(ibdev, &req->sqe, sizeof(struct nvme_command),
320 req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
322 if (IS_ERR(req->mr)) {
323 ret = PTR_ERR(req->mr);
332 nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
337 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
338 unsigned int hctx_idx)
340 struct nvme_rdma_ctrl *ctrl = data;
341 struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
343 BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
345 hctx->driver_data = queue;
349 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
350 unsigned int hctx_idx)
352 struct nvme_rdma_ctrl *ctrl = data;
353 struct nvme_rdma_queue *queue = &ctrl->queues[0];
355 BUG_ON(hctx_idx != 0);
357 hctx->driver_data = queue;
361 static void nvme_rdma_free_dev(struct kref *ref)
363 struct nvme_rdma_device *ndev =
364 container_of(ref, struct nvme_rdma_device, ref);
366 mutex_lock(&device_list_mutex);
367 list_del(&ndev->entry);
368 mutex_unlock(&device_list_mutex);
370 ib_dealloc_pd(ndev->pd);
374 static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
376 kref_put(&dev->ref, nvme_rdma_free_dev);
379 static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
381 return kref_get_unless_zero(&dev->ref);
384 static struct nvme_rdma_device *
385 nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
387 struct nvme_rdma_device *ndev;
389 mutex_lock(&device_list_mutex);
390 list_for_each_entry(ndev, &device_list, entry) {
391 if (ndev->dev->node_guid == cm_id->device->node_guid &&
392 nvme_rdma_dev_get(ndev))
396 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
400 ndev->dev = cm_id->device;
401 kref_init(&ndev->ref);
403 ndev->pd = ib_alloc_pd(ndev->dev,
404 register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
405 if (IS_ERR(ndev->pd))
408 if (!(ndev->dev->attrs.device_cap_flags &
409 IB_DEVICE_MEM_MGT_EXTENSIONS)) {
410 dev_err(&ndev->dev->dev,
411 "Memory registrations not supported.\n");
415 list_add(&ndev->entry, &device_list);
417 mutex_unlock(&device_list_mutex);
421 ib_dealloc_pd(ndev->pd);
425 mutex_unlock(&device_list_mutex);
429 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
431 struct nvme_rdma_device *dev = queue->device;
432 struct ib_device *ibdev = dev->dev;
434 rdma_destroy_qp(queue->cm_id);
435 ib_free_cq(queue->ib_cq);
437 nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
438 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
440 nvme_rdma_dev_put(dev);
443 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
445 struct ib_device *ibdev;
446 const int send_wr_factor = 3; /* MR, SEND, INV */
447 const int cq_factor = send_wr_factor + 1; /* + RECV */
448 int comp_vector, idx = nvme_rdma_queue_idx(queue);
451 queue->device = nvme_rdma_find_get_device(queue->cm_id);
452 if (!queue->device) {
453 dev_err(queue->cm_id->device->dev.parent,
454 "no client data found!\n");
455 return -ECONNREFUSED;
457 ibdev = queue->device->dev;
460 * Spread I/O queues completion vectors according their queue index.
461 * Admin queues can always go on completion vector 0.
463 comp_vector = idx == 0 ? idx : idx - 1;
465 /* +1 for ib_stop_cq */
466 queue->ib_cq = ib_alloc_cq(ibdev, queue,
467 cq_factor * queue->queue_size + 1,
468 comp_vector, IB_POLL_SOFTIRQ);
469 if (IS_ERR(queue->ib_cq)) {
470 ret = PTR_ERR(queue->ib_cq);
474 ret = nvme_rdma_create_qp(queue, send_wr_factor);
476 goto out_destroy_ib_cq;
478 queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
479 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
480 if (!queue->rsp_ring) {
488 rdma_destroy_qp(queue->cm_id);
490 ib_free_cq(queue->ib_cq);
492 nvme_rdma_dev_put(queue->device);
496 static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
497 int idx, size_t queue_size)
499 struct nvme_rdma_queue *queue;
500 struct sockaddr *src_addr = NULL;
503 queue = &ctrl->queues[idx];
505 init_completion(&queue->cm_done);
508 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
510 queue->cmnd_capsule_len = sizeof(struct nvme_command);
512 queue->queue_size = queue_size;
513 atomic_set(&queue->sig_count, 0);
515 queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
516 RDMA_PS_TCP, IB_QPT_RC);
517 if (IS_ERR(queue->cm_id)) {
518 dev_info(ctrl->ctrl.device,
519 "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
520 return PTR_ERR(queue->cm_id);
523 if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
524 src_addr = (struct sockaddr *)&ctrl->src_addr;
526 queue->cm_error = -ETIMEDOUT;
527 ret = rdma_resolve_addr(queue->cm_id, src_addr,
528 (struct sockaddr *)&ctrl->addr,
529 NVME_RDMA_CONNECT_TIMEOUT_MS);
531 dev_info(ctrl->ctrl.device,
532 "rdma_resolve_addr failed (%d).\n", ret);
533 goto out_destroy_cm_id;
536 ret = nvme_rdma_wait_for_cm(queue);
538 dev_info(ctrl->ctrl.device,
539 "rdma connection establishment failed (%d)\n", ret);
540 goto out_destroy_cm_id;
543 set_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags);
548 rdma_destroy_id(queue->cm_id);
552 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
554 if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
557 rdma_disconnect(queue->cm_id);
558 ib_drain_qp(queue->qp);
561 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
563 if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
566 if (nvme_rdma_queue_idx(queue) == 0) {
567 nvme_rdma_free_qe(queue->device->dev,
568 &queue->ctrl->async_event_sqe,
569 sizeof(struct nvme_command), DMA_TO_DEVICE);
572 nvme_rdma_destroy_queue_ib(queue);
573 rdma_destroy_id(queue->cm_id);
576 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
580 for (i = 1; i < ctrl->ctrl.queue_count; i++)
581 nvme_rdma_free_queue(&ctrl->queues[i]);
584 static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
588 for (i = 1; i < ctrl->ctrl.queue_count; i++)
589 nvme_rdma_stop_queue(&ctrl->queues[i]);
592 static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
597 ret = nvmf_connect_io_queue(&ctrl->ctrl, idx);
599 ret = nvmf_connect_admin_queue(&ctrl->ctrl);
602 set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[idx].flags);
604 dev_info(ctrl->ctrl.device,
605 "failed to connect queue: %d ret=%d\n", idx, ret);
609 static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl)
613 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
614 ret = nvme_rdma_start_queue(ctrl, i);
616 goto out_stop_queues;
622 for (i--; i >= 1; i--)
623 nvme_rdma_stop_queue(&ctrl->queues[i]);
627 static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
629 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
630 struct ib_device *ibdev = ctrl->device->dev;
631 unsigned int nr_io_queues;
634 nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
637 * we map queues according to the device irq vectors for
638 * optimal locality so we don't need more queues than
639 * completion vectors.
641 nr_io_queues = min_t(unsigned int, nr_io_queues,
642 ibdev->num_comp_vectors);
644 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
648 ctrl->ctrl.queue_count = nr_io_queues + 1;
649 if (ctrl->ctrl.queue_count < 2)
652 dev_info(ctrl->ctrl.device,
653 "creating %d I/O queues.\n", nr_io_queues);
655 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
656 ret = nvme_rdma_alloc_queue(ctrl, i,
657 ctrl->ctrl.sqsize + 1);
659 goto out_free_queues;
665 for (i--; i >= 1; i--)
666 nvme_rdma_free_queue(&ctrl->queues[i]);
671 static void nvme_rdma_free_tagset(struct nvme_ctrl *nctrl,
672 struct blk_mq_tag_set *set)
674 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
676 blk_mq_free_tag_set(set);
677 nvme_rdma_dev_put(ctrl->device);
680 static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl,
683 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
684 struct blk_mq_tag_set *set;
688 set = &ctrl->admin_tag_set;
689 memset(set, 0, sizeof(*set));
690 set->ops = &nvme_rdma_admin_mq_ops;
691 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
692 set->reserved_tags = 2; /* connect + keep-alive */
693 set->numa_node = NUMA_NO_NODE;
694 set->cmd_size = sizeof(struct nvme_rdma_request) +
695 SG_CHUNK_SIZE * sizeof(struct scatterlist);
696 set->driver_data = ctrl;
697 set->nr_hw_queues = 1;
698 set->timeout = ADMIN_TIMEOUT;
699 set->flags = BLK_MQ_F_NO_SCHED;
701 set = &ctrl->tag_set;
702 memset(set, 0, sizeof(*set));
703 set->ops = &nvme_rdma_mq_ops;
704 set->queue_depth = nctrl->opts->queue_size;
705 set->reserved_tags = 1; /* fabric connect */
706 set->numa_node = NUMA_NO_NODE;
707 set->flags = BLK_MQ_F_SHOULD_MERGE;
708 set->cmd_size = sizeof(struct nvme_rdma_request) +
709 SG_CHUNK_SIZE * sizeof(struct scatterlist);
710 set->driver_data = ctrl;
711 set->nr_hw_queues = nctrl->queue_count - 1;
712 set->timeout = NVME_IO_TIMEOUT;
715 ret = blk_mq_alloc_tag_set(set);
720 * We need a reference on the device as long as the tag_set is alive,
721 * as the MRs in the request structures need a valid ib_device.
723 ret = nvme_rdma_dev_get(ctrl->device);
726 goto out_free_tagset;
732 blk_mq_free_tag_set(set);
737 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
740 nvme_rdma_stop_queue(&ctrl->queues[0]);
742 blk_cleanup_queue(ctrl->ctrl.admin_q);
743 nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
745 nvme_rdma_free_queue(&ctrl->queues[0]);
748 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
753 error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
757 ctrl->device = ctrl->queues[0].device;
759 ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
760 ctrl->device->dev->attrs.max_fast_reg_page_list_len);
763 ctrl->ctrl.admin_tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, true);
764 if (IS_ERR(ctrl->ctrl.admin_tagset)) {
765 error = PTR_ERR(ctrl->ctrl.admin_tagset);
769 ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
770 if (IS_ERR(ctrl->ctrl.admin_q)) {
771 error = PTR_ERR(ctrl->ctrl.admin_q);
772 goto out_free_tagset;
775 error = nvme_reinit_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
780 error = nvme_rdma_start_queue(ctrl, 0);
782 goto out_cleanup_queue;
784 error = ctrl->ctrl.ops->reg_read64(&ctrl->ctrl, NVME_REG_CAP,
787 dev_err(ctrl->ctrl.device,
788 "prop_get NVME_REG_CAP failed\n");
789 goto out_cleanup_queue;
793 min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
795 error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
797 goto out_cleanup_queue;
799 ctrl->ctrl.max_hw_sectors =
800 (ctrl->max_fr_pages - 1) << (ilog2(SZ_4K) - 9);
802 error = nvme_init_identify(&ctrl->ctrl);
804 goto out_cleanup_queue;
806 error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
807 &ctrl->async_event_sqe, sizeof(struct nvme_command),
810 goto out_cleanup_queue;
816 blk_cleanup_queue(ctrl->ctrl.admin_q);
819 nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
821 nvme_rdma_free_queue(&ctrl->queues[0]);
825 static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,
828 nvme_rdma_stop_io_queues(ctrl);
830 blk_cleanup_queue(ctrl->ctrl.connect_q);
831 nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
833 nvme_rdma_free_io_queues(ctrl);
836 static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
840 ret = nvme_rdma_alloc_io_queues(ctrl);
845 ctrl->ctrl.tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, false);
846 if (IS_ERR(ctrl->ctrl.tagset)) {
847 ret = PTR_ERR(ctrl->ctrl.tagset);
848 goto out_free_io_queues;
851 ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
852 if (IS_ERR(ctrl->ctrl.connect_q)) {
853 ret = PTR_ERR(ctrl->ctrl.connect_q);
854 goto out_free_tag_set;
857 ret = nvme_reinit_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
859 goto out_free_io_queues;
861 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
862 ctrl->ctrl.queue_count - 1);
865 ret = nvme_rdma_start_io_queues(ctrl);
867 goto out_cleanup_connect_q;
871 out_cleanup_connect_q:
873 blk_cleanup_queue(ctrl->ctrl.connect_q);
876 nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
878 nvme_rdma_free_io_queues(ctrl);
882 static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
884 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
886 if (list_empty(&ctrl->list))
889 mutex_lock(&nvme_rdma_ctrl_mutex);
890 list_del(&ctrl->list);
891 mutex_unlock(&nvme_rdma_ctrl_mutex);
894 nvmf_free_options(nctrl->opts);
899 static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
901 /* If we are resetting/deleting then do nothing */
902 if (ctrl->ctrl.state != NVME_CTRL_RECONNECTING) {
903 WARN_ON_ONCE(ctrl->ctrl.state == NVME_CTRL_NEW ||
904 ctrl->ctrl.state == NVME_CTRL_LIVE);
908 if (nvmf_should_reconnect(&ctrl->ctrl)) {
909 dev_info(ctrl->ctrl.device, "Reconnecting in %d seconds...\n",
910 ctrl->ctrl.opts->reconnect_delay);
911 queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
912 ctrl->ctrl.opts->reconnect_delay * HZ);
914 dev_info(ctrl->ctrl.device, "Removing controller...\n");
915 nvme_delete_ctrl(&ctrl->ctrl);
919 static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
921 struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
922 struct nvme_rdma_ctrl, reconnect_work);
926 ++ctrl->ctrl.nr_reconnects;
928 ret = nvme_rdma_configure_admin_queue(ctrl, false);
932 if (ctrl->ctrl.queue_count > 1) {
933 ret = nvme_rdma_configure_io_queues(ctrl, false);
938 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
940 /* state change failure is ok if we're in DELETING state */
941 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
945 nvme_start_ctrl(&ctrl->ctrl);
947 dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
948 ctrl->ctrl.nr_reconnects);
950 ctrl->ctrl.nr_reconnects = 0;
955 nvme_rdma_destroy_admin_queue(ctrl, false);
957 dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
958 ctrl->ctrl.nr_reconnects);
959 nvme_rdma_reconnect_or_remove(ctrl);
962 static void nvme_rdma_error_recovery_work(struct work_struct *work)
964 struct nvme_rdma_ctrl *ctrl = container_of(work,
965 struct nvme_rdma_ctrl, err_work);
967 nvme_stop_keep_alive(&ctrl->ctrl);
969 if (ctrl->ctrl.queue_count > 1) {
970 nvme_stop_queues(&ctrl->ctrl);
971 blk_mq_tagset_busy_iter(&ctrl->tag_set,
972 nvme_cancel_request, &ctrl->ctrl);
973 nvme_rdma_destroy_io_queues(ctrl, false);
976 blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
977 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
978 nvme_cancel_request, &ctrl->ctrl);
979 nvme_rdma_destroy_admin_queue(ctrl, false);
982 * queues are not a live anymore, so restart the queues to fail fast
985 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
986 nvme_start_queues(&ctrl->ctrl);
988 nvme_rdma_reconnect_or_remove(ctrl);
991 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
993 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RECONNECTING))
996 queue_work(nvme_wq, &ctrl->err_work);
999 static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
1002 struct nvme_rdma_queue *queue = cq->cq_context;
1003 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1005 if (ctrl->ctrl.state == NVME_CTRL_LIVE)
1006 dev_info(ctrl->ctrl.device,
1007 "%s for CQE 0x%p failed with status %s (%d)\n",
1009 ib_wc_status_msg(wc->status), wc->status);
1010 nvme_rdma_error_recovery(ctrl);
1013 static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
1015 if (unlikely(wc->status != IB_WC_SUCCESS))
1016 nvme_rdma_wr_error(cq, wc, "MEMREG");
1019 static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
1021 if (unlikely(wc->status != IB_WC_SUCCESS))
1022 nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
1025 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
1026 struct nvme_rdma_request *req)
1028 struct ib_send_wr *bad_wr;
1029 struct ib_send_wr wr = {
1030 .opcode = IB_WR_LOCAL_INV,
1034 .ex.invalidate_rkey = req->mr->rkey,
1037 req->reg_cqe.done = nvme_rdma_inv_rkey_done;
1038 wr.wr_cqe = &req->reg_cqe;
1040 return ib_post_send(queue->qp, &wr, &bad_wr);
1043 static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
1046 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1047 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1048 struct nvme_rdma_device *dev = queue->device;
1049 struct ib_device *ibdev = dev->dev;
1052 if (!blk_rq_bytes(rq))
1055 if (req->mr->need_inval && test_bit(NVME_RDMA_Q_LIVE, &req->queue->flags)) {
1056 res = nvme_rdma_inv_rkey(queue, req);
1057 if (unlikely(res < 0)) {
1058 dev_err(ctrl->ctrl.device,
1059 "Queueing INV WR for rkey %#x failed (%d)\n",
1060 req->mr->rkey, res);
1061 nvme_rdma_error_recovery(queue->ctrl);
1065 ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
1066 req->nents, rq_data_dir(rq) ==
1067 WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1069 nvme_cleanup_cmd(rq);
1070 sg_free_table_chained(&req->sg_table, true);
1073 static int nvme_rdma_set_sg_null(struct nvme_command *c)
1075 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1078 put_unaligned_le24(0, sg->length);
1079 put_unaligned_le32(0, sg->key);
1080 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1084 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
1085 struct nvme_rdma_request *req, struct nvme_command *c)
1087 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
1089 req->sge[1].addr = sg_dma_address(req->sg_table.sgl);
1090 req->sge[1].length = sg_dma_len(req->sg_table.sgl);
1091 req->sge[1].lkey = queue->device->pd->local_dma_lkey;
1093 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
1094 sg->length = cpu_to_le32(sg_dma_len(req->sg_table.sgl));
1095 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
1097 req->inline_data = true;
1102 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
1103 struct nvme_rdma_request *req, struct nvme_command *c)
1105 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1107 sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
1108 put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
1109 put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
1110 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1114 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
1115 struct nvme_rdma_request *req, struct nvme_command *c,
1118 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1122 * Align the MR to a 4K page size to match the ctrl page size and
1123 * the block virtual boundary.
1125 nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, SZ_4K);
1126 if (unlikely(nr < count)) {
1132 ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
1134 req->reg_cqe.done = nvme_rdma_memreg_done;
1135 memset(&req->reg_wr, 0, sizeof(req->reg_wr));
1136 req->reg_wr.wr.opcode = IB_WR_REG_MR;
1137 req->reg_wr.wr.wr_cqe = &req->reg_cqe;
1138 req->reg_wr.wr.num_sge = 0;
1139 req->reg_wr.mr = req->mr;
1140 req->reg_wr.key = req->mr->rkey;
1141 req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
1142 IB_ACCESS_REMOTE_READ |
1143 IB_ACCESS_REMOTE_WRITE;
1145 req->mr->need_inval = true;
1147 sg->addr = cpu_to_le64(req->mr->iova);
1148 put_unaligned_le24(req->mr->length, sg->length);
1149 put_unaligned_le32(req->mr->rkey, sg->key);
1150 sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
1151 NVME_SGL_FMT_INVALIDATE;
1156 static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
1157 struct request *rq, struct nvme_command *c)
1159 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1160 struct nvme_rdma_device *dev = queue->device;
1161 struct ib_device *ibdev = dev->dev;
1165 req->inline_data = false;
1166 req->mr->need_inval = false;
1168 c->common.flags |= NVME_CMD_SGL_METABUF;
1170 if (!blk_rq_bytes(rq))
1171 return nvme_rdma_set_sg_null(c);
1173 req->sg_table.sgl = req->first_sgl;
1174 ret = sg_alloc_table_chained(&req->sg_table,
1175 blk_rq_nr_phys_segments(rq), req->sg_table.sgl);
1179 req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
1181 count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents,
1182 rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1183 if (unlikely(count <= 0)) {
1184 sg_free_table_chained(&req->sg_table, true);
1189 if (rq_data_dir(rq) == WRITE && nvme_rdma_queue_idx(queue) &&
1190 blk_rq_payload_bytes(rq) <=
1191 nvme_rdma_inline_data_size(queue))
1192 return nvme_rdma_map_sg_inline(queue, req, c);
1194 if (dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)
1195 return nvme_rdma_map_sg_single(queue, req, c);
1198 return nvme_rdma_map_sg_fr(queue, req, c, count);
1201 static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1203 if (unlikely(wc->status != IB_WC_SUCCESS))
1204 nvme_rdma_wr_error(cq, wc, "SEND");
1208 * We want to signal completion at least every queue depth/2. This returns the
1209 * largest power of two that is not above half of (queue size + 1) to optimize
1210 * (avoid divisions).
1212 static inline bool nvme_rdma_queue_sig_limit(struct nvme_rdma_queue *queue)
1214 int limit = 1 << ilog2((queue->queue_size + 1) / 2);
1216 return (atomic_inc_return(&queue->sig_count) & (limit - 1)) == 0;
1219 static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1220 struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
1221 struct ib_send_wr *first, bool flush)
1223 struct ib_send_wr wr, *bad_wr;
1226 sge->addr = qe->dma;
1227 sge->length = sizeof(struct nvme_command),
1228 sge->lkey = queue->device->pd->local_dma_lkey;
1230 qe->cqe.done = nvme_rdma_send_done;
1233 wr.wr_cqe = &qe->cqe;
1235 wr.num_sge = num_sge;
1236 wr.opcode = IB_WR_SEND;
1240 * Unsignalled send completions are another giant desaster in the
1241 * IB Verbs spec: If we don't regularly post signalled sends
1242 * the send queue will fill up and only a QP reset will rescue us.
1243 * Would have been way to obvious to handle this in hardware or
1244 * at least the RDMA stack..
1246 * Always signal the flushes. The magic request used for the flush
1247 * sequencer is not allocated in our driver's tagset and it's
1248 * triggered to be freed by blk_cleanup_queue(). So we need to
1249 * always mark it as signaled to ensure that the "wr_cqe", which is
1250 * embedded in request's payload, is not freed when __ib_process_cq()
1251 * calls wr_cqe->done().
1253 if (nvme_rdma_queue_sig_limit(queue) || flush)
1254 wr.send_flags |= IB_SEND_SIGNALED;
1261 ret = ib_post_send(queue->qp, first, &bad_wr);
1262 if (unlikely(ret)) {
1263 dev_err(queue->ctrl->ctrl.device,
1264 "%s failed with error code %d\n", __func__, ret);
1269 static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1270 struct nvme_rdma_qe *qe)
1272 struct ib_recv_wr wr, *bad_wr;
1276 list.addr = qe->dma;
1277 list.length = sizeof(struct nvme_completion);
1278 list.lkey = queue->device->pd->local_dma_lkey;
1280 qe->cqe.done = nvme_rdma_recv_done;
1283 wr.wr_cqe = &qe->cqe;
1287 ret = ib_post_recv(queue->qp, &wr, &bad_wr);
1288 if (unlikely(ret)) {
1289 dev_err(queue->ctrl->ctrl.device,
1290 "%s failed with error code %d\n", __func__, ret);
1295 static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1297 u32 queue_idx = nvme_rdma_queue_idx(queue);
1300 return queue->ctrl->admin_tag_set.tags[queue_idx];
1301 return queue->ctrl->tag_set.tags[queue_idx - 1];
1304 static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg)
1306 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1307 struct nvme_rdma_queue *queue = &ctrl->queues[0];
1308 struct ib_device *dev = queue->device->dev;
1309 struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1310 struct nvme_command *cmd = sqe->data;
1314 ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1316 memset(cmd, 0, sizeof(*cmd));
1317 cmd->common.opcode = nvme_admin_async_event;
1318 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1319 cmd->common.flags |= NVME_CMD_SGL_METABUF;
1320 nvme_rdma_set_sg_null(cmd);
1322 ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1325 ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL, false);
1329 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1330 struct nvme_completion *cqe, struct ib_wc *wc, int tag)
1333 struct nvme_rdma_request *req;
1336 rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
1338 dev_err(queue->ctrl->ctrl.device,
1339 "tag 0x%x on QP %#x not found\n",
1340 cqe->command_id, queue->qp->qp_num);
1341 nvme_rdma_error_recovery(queue->ctrl);
1344 req = blk_mq_rq_to_pdu(rq);
1349 if ((wc->wc_flags & IB_WC_WITH_INVALIDATE) &&
1350 wc->ex.invalidate_rkey == req->mr->rkey)
1351 req->mr->need_inval = false;
1353 nvme_end_request(rq, cqe->status, cqe->result);
1357 static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
1359 struct nvme_rdma_qe *qe =
1360 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1361 struct nvme_rdma_queue *queue = cq->cq_context;
1362 struct ib_device *ibdev = queue->device->dev;
1363 struct nvme_completion *cqe = qe->data;
1364 const size_t len = sizeof(struct nvme_completion);
1367 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1368 nvme_rdma_wr_error(cq, wc, "RECV");
1372 ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1374 * AEN requests are special as they don't time out and can
1375 * survive any kind of queue freeze and often don't respond to
1376 * aborts. We don't even bother to allocate a struct request
1377 * for them but rather special case them here.
1379 if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1380 cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH))
1381 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1384 ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
1385 ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1387 nvme_rdma_post_recv(queue, qe);
1391 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1393 __nvme_rdma_recv_done(cq, wc, -1);
1396 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1400 for (i = 0; i < queue->queue_size; i++) {
1401 ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1403 goto out_destroy_queue_ib;
1408 out_destroy_queue_ib:
1409 nvme_rdma_destroy_queue_ib(queue);
1413 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1414 struct rdma_cm_event *ev)
1416 struct rdma_cm_id *cm_id = queue->cm_id;
1417 int status = ev->status;
1418 const char *rej_msg;
1419 const struct nvme_rdma_cm_rej *rej_data;
1422 rej_msg = rdma_reject_msg(cm_id, status);
1423 rej_data = rdma_consumer_reject_data(cm_id, ev, &rej_data_len);
1425 if (rej_data && rej_data_len >= sizeof(u16)) {
1426 u16 sts = le16_to_cpu(rej_data->sts);
1428 dev_err(queue->ctrl->ctrl.device,
1429 "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1430 status, rej_msg, sts, nvme_rdma_cm_msg(sts));
1432 dev_err(queue->ctrl->ctrl.device,
1433 "Connect rejected: status %d (%s).\n", status, rej_msg);
1439 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1443 ret = nvme_rdma_create_queue_ib(queue);
1447 ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1449 dev_err(queue->ctrl->ctrl.device,
1450 "rdma_resolve_route failed (%d).\n",
1452 goto out_destroy_queue;
1458 nvme_rdma_destroy_queue_ib(queue);
1462 static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1464 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1465 struct rdma_conn_param param = { };
1466 struct nvme_rdma_cm_req priv = { };
1469 param.qp_num = queue->qp->qp_num;
1470 param.flow_control = 1;
1472 param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
1473 /* maximum retry count */
1474 param.retry_count = 7;
1475 param.rnr_retry_count = 7;
1476 param.private_data = &priv;
1477 param.private_data_len = sizeof(priv);
1479 priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1480 priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
1482 * set the admin queue depth to the minimum size
1483 * specified by the Fabrics standard.
1485 if (priv.qid == 0) {
1486 priv.hrqsize = cpu_to_le16(NVME_AQ_DEPTH);
1487 priv.hsqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
1490 * current interpretation of the fabrics spec
1491 * is at minimum you make hrqsize sqsize+1, or a
1492 * 1's based representation of sqsize.
1494 priv.hrqsize = cpu_to_le16(queue->queue_size);
1495 priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
1498 ret = rdma_connect(queue->cm_id, ¶m);
1500 dev_err(ctrl->ctrl.device,
1501 "rdma_connect failed (%d).\n", ret);
1502 goto out_destroy_queue_ib;
1507 out_destroy_queue_ib:
1508 nvme_rdma_destroy_queue_ib(queue);
1512 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1513 struct rdma_cm_event *ev)
1515 struct nvme_rdma_queue *queue = cm_id->context;
1518 dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1519 rdma_event_msg(ev->event), ev->event,
1522 switch (ev->event) {
1523 case RDMA_CM_EVENT_ADDR_RESOLVED:
1524 cm_error = nvme_rdma_addr_resolved(queue);
1526 case RDMA_CM_EVENT_ROUTE_RESOLVED:
1527 cm_error = nvme_rdma_route_resolved(queue);
1529 case RDMA_CM_EVENT_ESTABLISHED:
1530 queue->cm_error = nvme_rdma_conn_established(queue);
1531 /* complete cm_done regardless of success/failure */
1532 complete(&queue->cm_done);
1534 case RDMA_CM_EVENT_REJECTED:
1535 nvme_rdma_destroy_queue_ib(queue);
1536 cm_error = nvme_rdma_conn_rejected(queue, ev);
1538 case RDMA_CM_EVENT_ROUTE_ERROR:
1539 case RDMA_CM_EVENT_CONNECT_ERROR:
1540 case RDMA_CM_EVENT_UNREACHABLE:
1541 nvme_rdma_destroy_queue_ib(queue);
1542 case RDMA_CM_EVENT_ADDR_ERROR:
1543 dev_dbg(queue->ctrl->ctrl.device,
1544 "CM error event %d\n", ev->event);
1545 cm_error = -ECONNRESET;
1547 case RDMA_CM_EVENT_DISCONNECTED:
1548 case RDMA_CM_EVENT_ADDR_CHANGE:
1549 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1550 dev_dbg(queue->ctrl->ctrl.device,
1551 "disconnect received - connection closed\n");
1552 nvme_rdma_error_recovery(queue->ctrl);
1554 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1555 /* device removal is handled via the ib_client API */
1558 dev_err(queue->ctrl->ctrl.device,
1559 "Unexpected RDMA CM event (%d)\n", ev->event);
1560 nvme_rdma_error_recovery(queue->ctrl);
1565 queue->cm_error = cm_error;
1566 complete(&queue->cm_done);
1572 static enum blk_eh_timer_return
1573 nvme_rdma_timeout(struct request *rq, bool reserved)
1575 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1577 dev_warn(req->queue->ctrl->ctrl.device,
1578 "I/O %d QID %d timeout, reset controller\n",
1579 rq->tag, nvme_rdma_queue_idx(req->queue));
1581 /* queue error recovery */
1582 nvme_rdma_error_recovery(req->queue->ctrl);
1584 /* fail with DNR on cmd timeout */
1585 nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR;
1587 return BLK_EH_HANDLED;
1591 * We cannot accept any other command until the Connect command has completed.
1593 static inline blk_status_t
1594 nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue, struct request *rq)
1596 if (unlikely(!test_bit(NVME_RDMA_Q_LIVE, &queue->flags))) {
1597 struct nvme_command *cmd = nvme_req(rq)->cmd;
1599 if (!blk_rq_is_passthrough(rq) ||
1600 cmd->common.opcode != nvme_fabrics_command ||
1601 cmd->fabrics.fctype != nvme_fabrics_type_connect) {
1603 * reconnecting state means transport disruption, which
1604 * can take a long time and even might fail permanently,
1605 * fail fast to give upper layers a chance to failover.
1606 * deleting state means that the ctrl will never accept
1607 * commands again, fail it permanently.
1609 if (queue->ctrl->ctrl.state == NVME_CTRL_RECONNECTING ||
1610 queue->ctrl->ctrl.state == NVME_CTRL_DELETING) {
1611 nvme_req(rq)->status = NVME_SC_ABORT_REQ;
1612 return BLK_STS_IOERR;
1614 return BLK_STS_RESOURCE; /* try again later */
1621 static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
1622 const struct blk_mq_queue_data *bd)
1624 struct nvme_ns *ns = hctx->queue->queuedata;
1625 struct nvme_rdma_queue *queue = hctx->driver_data;
1626 struct request *rq = bd->rq;
1627 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1628 struct nvme_rdma_qe *sqe = &req->sqe;
1629 struct nvme_command *c = sqe->data;
1631 struct ib_device *dev;
1635 WARN_ON_ONCE(rq->tag < 0);
1637 ret = nvme_rdma_queue_is_ready(queue, rq);
1641 dev = queue->device->dev;
1642 ib_dma_sync_single_for_cpu(dev, sqe->dma,
1643 sizeof(struct nvme_command), DMA_TO_DEVICE);
1645 ret = nvme_setup_cmd(ns, rq, c);
1649 blk_mq_start_request(rq);
1651 err = nvme_rdma_map_data(queue, rq, c);
1652 if (unlikely(err < 0)) {
1653 dev_err(queue->ctrl->ctrl.device,
1654 "Failed to map data (%d)\n", err);
1655 nvme_cleanup_cmd(rq);
1659 ib_dma_sync_single_for_device(dev, sqe->dma,
1660 sizeof(struct nvme_command), DMA_TO_DEVICE);
1662 if (req_op(rq) == REQ_OP_FLUSH)
1664 err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1665 req->mr->need_inval ? &req->reg_wr.wr : NULL, flush);
1666 if (unlikely(err)) {
1667 nvme_rdma_unmap_data(queue, rq);
1673 if (err == -ENOMEM || err == -EAGAIN)
1674 return BLK_STS_RESOURCE;
1675 return BLK_STS_IOERR;
1678 static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
1680 struct nvme_rdma_queue *queue = hctx->driver_data;
1681 struct ib_cq *cq = queue->ib_cq;
1685 while (ib_poll_cq(cq, 1, &wc) > 0) {
1686 struct ib_cqe *cqe = wc.wr_cqe;
1689 if (cqe->done == nvme_rdma_recv_done)
1690 found |= __nvme_rdma_recv_done(cq, &wc, tag);
1699 static void nvme_rdma_complete_rq(struct request *rq)
1701 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1703 nvme_rdma_unmap_data(req->queue, rq);
1704 nvme_complete_rq(rq);
1707 static int nvme_rdma_map_queues(struct blk_mq_tag_set *set)
1709 struct nvme_rdma_ctrl *ctrl = set->driver_data;
1711 return blk_mq_rdma_map_queues(set, ctrl->device->dev, 0);
1714 static const struct blk_mq_ops nvme_rdma_mq_ops = {
1715 .queue_rq = nvme_rdma_queue_rq,
1716 .complete = nvme_rdma_complete_rq,
1717 .init_request = nvme_rdma_init_request,
1718 .exit_request = nvme_rdma_exit_request,
1719 .init_hctx = nvme_rdma_init_hctx,
1720 .poll = nvme_rdma_poll,
1721 .timeout = nvme_rdma_timeout,
1722 .map_queues = nvme_rdma_map_queues,
1725 static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
1726 .queue_rq = nvme_rdma_queue_rq,
1727 .complete = nvme_rdma_complete_rq,
1728 .init_request = nvme_rdma_init_request,
1729 .exit_request = nvme_rdma_exit_request,
1730 .init_hctx = nvme_rdma_init_admin_hctx,
1731 .timeout = nvme_rdma_timeout,
1734 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1736 cancel_work_sync(&ctrl->err_work);
1737 cancel_delayed_work_sync(&ctrl->reconnect_work);
1739 if (ctrl->ctrl.queue_count > 1) {
1740 nvme_stop_queues(&ctrl->ctrl);
1741 blk_mq_tagset_busy_iter(&ctrl->tag_set,
1742 nvme_cancel_request, &ctrl->ctrl);
1743 nvme_rdma_destroy_io_queues(ctrl, shutdown);
1747 nvme_shutdown_ctrl(&ctrl->ctrl);
1749 nvme_disable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
1751 blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
1752 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1753 nvme_cancel_request, &ctrl->ctrl);
1754 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
1755 nvme_rdma_destroy_admin_queue(ctrl, shutdown);
1758 static void nvme_rdma_delete_ctrl(struct nvme_ctrl *ctrl)
1760 nvme_rdma_shutdown_ctrl(to_rdma_ctrl(ctrl), true);
1763 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1765 struct nvme_rdma_ctrl *ctrl =
1766 container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
1770 nvme_stop_ctrl(&ctrl->ctrl);
1771 nvme_rdma_shutdown_ctrl(ctrl, false);
1773 ret = nvme_rdma_configure_admin_queue(ctrl, false);
1777 if (ctrl->ctrl.queue_count > 1) {
1778 ret = nvme_rdma_configure_io_queues(ctrl, false);
1783 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1785 /* state change failure is ok if we're in DELETING state */
1786 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
1790 nvme_start_ctrl(&ctrl->ctrl);
1795 dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1796 nvme_remove_namespaces(&ctrl->ctrl);
1797 nvme_rdma_shutdown_ctrl(ctrl, true);
1798 nvme_uninit_ctrl(&ctrl->ctrl);
1799 nvme_put_ctrl(&ctrl->ctrl);
1802 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1804 .module = THIS_MODULE,
1805 .flags = NVME_F_FABRICS,
1806 .reg_read32 = nvmf_reg_read32,
1807 .reg_read64 = nvmf_reg_read64,
1808 .reg_write32 = nvmf_reg_write32,
1809 .free_ctrl = nvme_rdma_free_ctrl,
1810 .submit_async_event = nvme_rdma_submit_async_event,
1811 .delete_ctrl = nvme_rdma_delete_ctrl,
1812 .get_address = nvmf_get_address,
1813 .reinit_request = nvme_rdma_reinit_request,
1817 __nvme_rdma_options_match(struct nvme_rdma_ctrl *ctrl,
1818 struct nvmf_ctrl_options *opts)
1820 char *stdport = __stringify(NVME_RDMA_IP_PORT);
1823 if (!nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts) ||
1824 strcmp(opts->traddr, ctrl->ctrl.opts->traddr))
1827 if (opts->mask & NVMF_OPT_TRSVCID &&
1828 ctrl->ctrl.opts->mask & NVMF_OPT_TRSVCID) {
1829 if (strcmp(opts->trsvcid, ctrl->ctrl.opts->trsvcid))
1831 } else if (opts->mask & NVMF_OPT_TRSVCID) {
1832 if (strcmp(opts->trsvcid, stdport))
1834 } else if (ctrl->ctrl.opts->mask & NVMF_OPT_TRSVCID) {
1835 if (strcmp(stdport, ctrl->ctrl.opts->trsvcid))
1838 /* else, it's a match as both have stdport. Fall to next checks */
1841 * checking the local address is rough. In most cases, one
1842 * is not specified and the host port is selected by the stack.
1844 * Assume no match if:
1845 * local address is specified and address is not the same
1846 * local address is not specified but remote is, or vice versa
1847 * (admin using specific host_traddr when it matters).
1849 if (opts->mask & NVMF_OPT_HOST_TRADDR &&
1850 ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR) {
1851 if (strcmp(opts->host_traddr, ctrl->ctrl.opts->host_traddr))
1853 } else if (opts->mask & NVMF_OPT_HOST_TRADDR ||
1854 ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
1857 * if neither controller had an host port specified, assume it's
1858 * a match as everything else matched.
1865 * Fails a connection request if it matches an existing controller
1866 * (association) with the same tuple:
1867 * <Host NQN, Host ID, local address, remote address, remote port, SUBSYS NQN>
1869 * if local address is not specified in the request, it will match an
1870 * existing controller with all the other parameters the same and no
1871 * local port address specified as well.
1873 * The ports don't need to be compared as they are intrinsically
1874 * already matched by the port pointers supplied.
1877 nvme_rdma_existing_controller(struct nvmf_ctrl_options *opts)
1879 struct nvme_rdma_ctrl *ctrl;
1882 mutex_lock(&nvme_rdma_ctrl_mutex);
1883 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
1884 found = __nvme_rdma_options_match(ctrl, opts);
1888 mutex_unlock(&nvme_rdma_ctrl_mutex);
1893 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
1894 struct nvmf_ctrl_options *opts)
1896 struct nvme_rdma_ctrl *ctrl;
1901 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1903 return ERR_PTR(-ENOMEM);
1904 ctrl->ctrl.opts = opts;
1905 INIT_LIST_HEAD(&ctrl->list);
1907 if (opts->mask & NVMF_OPT_TRSVCID)
1908 port = opts->trsvcid;
1910 port = __stringify(NVME_RDMA_IP_PORT);
1912 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1913 opts->traddr, port, &ctrl->addr);
1915 pr_err("malformed address passed: %s:%s\n", opts->traddr, port);
1919 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
1920 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1921 opts->host_traddr, NULL, &ctrl->src_addr);
1923 pr_err("malformed src address passed: %s\n",
1929 if (!opts->duplicate_connect && nvme_rdma_existing_controller(opts)) {
1934 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
1935 0 /* no quirks, we're perfect! */);
1939 INIT_DELAYED_WORK(&ctrl->reconnect_work,
1940 nvme_rdma_reconnect_ctrl_work);
1941 INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
1942 INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
1944 ctrl->ctrl.queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
1945 ctrl->ctrl.sqsize = opts->queue_size - 1;
1946 ctrl->ctrl.kato = opts->kato;
1949 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
1952 goto out_uninit_ctrl;
1954 ret = nvme_rdma_configure_admin_queue(ctrl, true);
1956 goto out_kfree_queues;
1958 /* sanity check icdoff */
1959 if (ctrl->ctrl.icdoff) {
1960 dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
1962 goto out_remove_admin_queue;
1965 /* sanity check keyed sgls */
1966 if (!(ctrl->ctrl.sgls & (1 << 20))) {
1967 dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not support\n");
1969 goto out_remove_admin_queue;
1972 if (opts->queue_size > ctrl->ctrl.maxcmd) {
1973 /* warn if maxcmd is lower than queue_size */
1974 dev_warn(ctrl->ctrl.device,
1975 "queue_size %zu > ctrl maxcmd %u, clamping down\n",
1976 opts->queue_size, ctrl->ctrl.maxcmd);
1977 opts->queue_size = ctrl->ctrl.maxcmd;
1980 if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
1981 /* warn if sqsize is lower than queue_size */
1982 dev_warn(ctrl->ctrl.device,
1983 "queue_size %zu > ctrl sqsize %u, clamping down\n",
1984 opts->queue_size, ctrl->ctrl.sqsize + 1);
1985 opts->queue_size = ctrl->ctrl.sqsize + 1;
1988 if (opts->nr_io_queues) {
1989 ret = nvme_rdma_configure_io_queues(ctrl, true);
1991 goto out_remove_admin_queue;
1994 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1995 WARN_ON_ONCE(!changed);
1997 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
1998 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2000 nvme_get_ctrl(&ctrl->ctrl);
2002 mutex_lock(&nvme_rdma_ctrl_mutex);
2003 list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
2004 mutex_unlock(&nvme_rdma_ctrl_mutex);
2006 nvme_start_ctrl(&ctrl->ctrl);
2010 out_remove_admin_queue:
2011 nvme_rdma_destroy_admin_queue(ctrl, true);
2013 kfree(ctrl->queues);
2015 nvme_uninit_ctrl(&ctrl->ctrl);
2016 nvme_put_ctrl(&ctrl->ctrl);
2019 return ERR_PTR(ret);
2022 return ERR_PTR(ret);
2025 static struct nvmf_transport_ops nvme_rdma_transport = {
2027 .required_opts = NVMF_OPT_TRADDR,
2028 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2029 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO,
2030 .create_ctrl = nvme_rdma_create_ctrl,
2033 static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
2035 struct nvme_rdma_ctrl *ctrl;
2037 /* Delete all controllers using this device */
2038 mutex_lock(&nvme_rdma_ctrl_mutex);
2039 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
2040 if (ctrl->device->dev != ib_device)
2042 dev_info(ctrl->ctrl.device,
2043 "Removing ctrl: NQN \"%s\", addr %pISp\n",
2044 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2045 nvme_delete_ctrl(&ctrl->ctrl);
2047 mutex_unlock(&nvme_rdma_ctrl_mutex);
2049 flush_workqueue(nvme_wq);
2052 static struct ib_client nvme_rdma_ib_client = {
2053 .name = "nvme_rdma",
2054 .remove = nvme_rdma_remove_one
2057 static int __init nvme_rdma_init_module(void)
2061 ret = ib_register_client(&nvme_rdma_ib_client);
2065 ret = nvmf_register_transport(&nvme_rdma_transport);
2067 goto err_unreg_client;
2072 ib_unregister_client(&nvme_rdma_ib_client);
2076 static void __exit nvme_rdma_cleanup_module(void)
2078 nvmf_unregister_transport(&nvme_rdma_transport);
2079 ib_unregister_client(&nvme_rdma_ib_client);
2082 module_init(nvme_rdma_init_module);
2083 module_exit(nvme_rdma_cleanup_module);
2085 MODULE_LICENSE("GPL v2");