1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * RDMA Network Block Driver
5 * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
6 * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
7 * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
11 #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
13 #include <linux/module.h>
14 #include <linux/blkdev.h>
15 #include <linux/hdreg.h>
16 #include <linux/scatterlist.h>
17 #include <linux/idr.h>
21 MODULE_DESCRIPTION("RDMA Network Block Device Client");
22 MODULE_LICENSE("GPL");
24 static int rnbd_client_major;
25 static DEFINE_IDA(index_ida);
26 static DEFINE_MUTEX(ida_lock);
27 static DEFINE_MUTEX(sess_lock);
28 static LIST_HEAD(sess_list);
31 * Maximum number of partitions an instance can have.
32 * 6 bits = 64 minors = 63 partitions (one minor is used for the device itself)
34 #define RNBD_PART_BITS 6
36 static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess)
38 return refcount_inc_not_zero(&sess->refcount);
41 static void free_sess(struct rnbd_clt_session *sess);
43 static void rnbd_clt_put_sess(struct rnbd_clt_session *sess)
47 if (refcount_dec_and_test(&sess->refcount))
51 static void rnbd_clt_put_dev(struct rnbd_clt_dev *dev)
55 if (!refcount_dec_and_test(&dev->refcount))
58 mutex_lock(&ida_lock);
59 ida_simple_remove(&index_ida, dev->clt_device_id);
60 mutex_unlock(&ida_lock);
61 kfree(dev->hw_queues);
62 rnbd_clt_put_sess(dev->sess);
63 mutex_destroy(&dev->lock);
67 static inline bool rnbd_clt_get_dev(struct rnbd_clt_dev *dev)
69 return refcount_inc_not_zero(&dev->refcount);
72 static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev,
73 const struct rnbd_msg_open_rsp *rsp)
75 struct rnbd_clt_session *sess = dev->sess;
77 if (!rsp->logical_block_size)
80 dev->device_id = le32_to_cpu(rsp->device_id);
81 dev->nsectors = le64_to_cpu(rsp->nsectors);
82 dev->logical_block_size = le16_to_cpu(rsp->logical_block_size);
83 dev->physical_block_size = le16_to_cpu(rsp->physical_block_size);
84 dev->max_write_same_sectors = le32_to_cpu(rsp->max_write_same_sectors);
85 dev->max_discard_sectors = le32_to_cpu(rsp->max_discard_sectors);
86 dev->discard_granularity = le32_to_cpu(rsp->discard_granularity);
87 dev->discard_alignment = le32_to_cpu(rsp->discard_alignment);
88 dev->secure_discard = le16_to_cpu(rsp->secure_discard);
89 dev->rotational = rsp->rotational;
91 dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE;
92 dev->max_segments = BMAX_SEGMENTS;
97 static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev,
100 rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n",
101 dev->nsectors, new_nsectors);
102 dev->nsectors = new_nsectors;
103 set_capacity(dev->gd, dev->nsectors);
104 revalidate_disk_size(dev->gd, true);
108 static int process_msg_open_rsp(struct rnbd_clt_dev *dev,
109 struct rnbd_msg_open_rsp *rsp)
113 mutex_lock(&dev->lock);
114 if (dev->dev_state == DEV_STATE_UNMAPPED) {
116 "Ignoring Open-Response message from server for unmapped device\n");
120 if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED) {
121 u64 nsectors = le64_to_cpu(rsp->nsectors);
124 * If the device was remapped and the size changed in the
125 * meantime we need to revalidate it
127 if (dev->nsectors != nsectors)
128 rnbd_clt_change_capacity(dev, nsectors);
129 rnbd_clt_info(dev, "Device online, device remapped successfully\n");
131 err = rnbd_clt_set_dev_attr(dev, rsp);
134 dev->dev_state = DEV_STATE_MAPPED;
137 mutex_unlock(&dev->lock);
142 int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, size_t newsize)
146 mutex_lock(&dev->lock);
147 if (dev->dev_state != DEV_STATE_MAPPED) {
148 pr_err("Failed to set new size of the device, device is not opened\n");
152 ret = rnbd_clt_change_capacity(dev, newsize);
155 mutex_unlock(&dev->lock);
160 static inline void rnbd_clt_dev_requeue(struct rnbd_queue *q)
162 if (WARN_ON(!q->hctx))
165 /* We can come here from interrupt, thus async=true */
166 blk_mq_run_hw_queue(q->hctx, true);
170 RNBD_DELAY_IFBUSY = -1,
174 * rnbd_get_cpu_qlist() - finds a list with HW queues to be rerun
175 * @sess: Session to find a queue for
176 * @cpu: Cpu to start the search from
179 * Each CPU has a list of HW queues, which needs to be rerun. If a list
180 * is not empty - it is marked with a bit. This function finds first
181 * set bit in a bitmap and returns corresponding CPU list.
183 static struct rnbd_cpu_qlist *
184 rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu)
188 /* Search from cpu to nr_cpu_ids */
189 bit = find_next_bit(sess->cpu_queues_bm, nr_cpu_ids, cpu);
190 if (bit < nr_cpu_ids) {
191 return per_cpu_ptr(sess->cpu_queues, bit);
192 } else if (cpu != 0) {
193 /* Search from 0 to cpu */
194 bit = find_next_bit(sess->cpu_queues_bm, cpu, 0);
196 return per_cpu_ptr(sess->cpu_queues, bit);
202 static inline int nxt_cpu(int cpu)
204 return (cpu + 1) % nr_cpu_ids;
208 * rnbd_rerun_if_needed() - rerun next queue marked as stopped
209 * @sess: Session to rerun a queue on
212 * Each CPU has it's own list of HW queues, which should be rerun.
213 * Function finds such list with HW queues, takes a list lock, picks up
214 * the first HW queue out of the list and requeues it.
217 * True if the queue was requeued, false otherwise.
222 static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess)
224 struct rnbd_queue *q = NULL;
225 struct rnbd_cpu_qlist *cpu_q;
230 * To keep fairness and not to let other queues starve we always
231 * try to wake up someone else in round-robin manner. That of course
232 * increases latency but queues always have a chance to be executed.
234 cpup = get_cpu_ptr(sess->cpu_rr);
235 for (cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(*cpup)); cpu_q;
236 cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(cpu_q->cpu))) {
237 if (!spin_trylock_irqsave(&cpu_q->requeue_lock, flags))
239 if (unlikely(!test_bit(cpu_q->cpu, sess->cpu_queues_bm)))
241 q = list_first_entry_or_null(&cpu_q->requeue_list,
242 typeof(*q), requeue_list);
245 list_del_init(&q->requeue_list);
246 clear_bit_unlock(0, &q->in_list);
248 if (list_empty(&cpu_q->requeue_list)) {
249 /* Clear bit if nothing is left */
251 clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
254 spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
261 * Saves the CPU that is going to be requeued on the per-cpu var. Just
262 * incrementing it doesn't work because rnbd_get_cpu_qlist() will
263 * always return the first CPU with something on the queue list when the
264 * value stored on the var is greater than the last CPU with something
269 put_cpu_var(sess->cpu_rr);
272 rnbd_clt_dev_requeue(q);
278 * rnbd_rerun_all_if_idle() - rerun all queues left in the list if
279 * session is idling (there are no requests
281 * @sess: Session to rerun the queues on
284 * This function tries to rerun all stopped queues if there are no
285 * requests in-flight anymore. This function tries to solve an obvious
286 * problem, when number of tags < than number of queues (hctx), which
287 * are stopped and put to sleep. If last permit, which has been just put,
288 * does not wake up all left queues (hctxs), IO requests hang forever.
290 * That can happen when all number of permits, say N, have been exhausted
291 * from one CPU, and we have many block devices per session, say M.
292 * Each block device has it's own queue (hctx) for each CPU, so eventually
293 * we can put that number of queues (hctxs) to sleep: M x nr_cpu_ids.
294 * If number of permits N < M x nr_cpu_ids finally we will get an IO hang.
296 * To avoid this hang last caller of rnbd_put_permit() (last caller is the
297 * one who observes sess->busy == 0) must wake up all remaining queues.
302 static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess)
307 requeued = rnbd_rerun_if_needed(sess);
308 } while (atomic_read(&sess->busy) == 0 && requeued);
311 static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess,
312 enum rtrs_clt_con_type con_type,
315 struct rtrs_permit *permit;
317 permit = rtrs_clt_get_permit(sess->rtrs, con_type,
318 wait ? RTRS_PERMIT_WAIT :
321 /* We have a subtle rare case here, when all permits can be
322 * consumed before busy counter increased. This is safe,
323 * because loser will get NULL as a permit, observe 0 busy
324 * counter and immediately restart the queue himself.
326 atomic_inc(&sess->busy);
331 static void rnbd_put_permit(struct rnbd_clt_session *sess,
332 struct rtrs_permit *permit)
334 rtrs_clt_put_permit(sess->rtrs, permit);
335 atomic_dec(&sess->busy);
336 /* Paired with rnbd_clt_dev_add_to_requeue(). Decrement first
337 * and then check queue bits.
339 smp_mb__after_atomic();
340 rnbd_rerun_all_if_idle(sess);
343 static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess,
344 enum rtrs_clt_con_type con_type,
348 struct rtrs_permit *permit;
350 permit = rnbd_get_permit(sess, con_type,
351 wait ? RTRS_PERMIT_WAIT :
353 if (unlikely(!permit))
355 iu = rtrs_permit_to_pdu(permit);
358 * 1st reference is dropped after finishing sending a "user" message,
359 * 2nd reference is dropped after confirmation with the response is
361 * 1st and 2nd can happen in any order, so the rnbd_iu should be
362 * released (rtrs_permit returned to ibbtrs) only leased after both
365 atomic_set(&iu->refcount, 2);
366 init_waitqueue_head(&iu->comp.wait);
367 iu->comp.errno = INT_MAX;
372 static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu)
374 if (atomic_dec_and_test(&iu->refcount))
375 rnbd_put_permit(sess, iu->permit);
378 static void rnbd_softirq_done_fn(struct request *rq)
380 struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
381 struct rnbd_clt_session *sess = dev->sess;
384 iu = blk_mq_rq_to_pdu(rq);
385 rnbd_put_permit(sess, iu->permit);
386 blk_mq_end_request(rq, errno_to_blk_status(iu->errno));
389 static void msg_io_conf(void *priv, int errno)
391 struct rnbd_iu *iu = priv;
392 struct rnbd_clt_dev *dev = iu->dev;
393 struct request *rq = iu->rq;
394 int rw = rq_data_dir(rq);
398 blk_mq_complete_request(rq);
401 rnbd_clt_info_rl(dev, "%s I/O failed with err: %d\n",
402 rw == READ ? "read" : "write", errno);
405 static void wake_up_iu_comp(struct rnbd_iu *iu, int errno)
407 iu->comp.errno = errno;
408 wake_up(&iu->comp.wait);
411 static void msg_conf(void *priv, int errno)
413 struct rnbd_iu *iu = priv;
416 schedule_work(&iu->work);
424 static int send_usr_msg(struct rtrs_clt *rtrs, int dir,
425 struct rnbd_iu *iu, struct kvec *vec,
426 size_t len, struct scatterlist *sg, unsigned int sg_len,
427 void (*conf)(struct work_struct *work),
428 int *errno, enum wait_type wait)
431 struct rtrs_clt_req_ops req_ops;
433 INIT_WORK(&iu->work, conf);
434 req_ops = (struct rtrs_clt_req_ops) {
438 err = rtrs_clt_request(dir, &req_ops, rtrs, iu->permit,
439 vec, 1, len, sg, sg_len);
441 wait_event(iu->comp.wait, iu->comp.errno != INT_MAX);
442 *errno = iu->comp.errno;
450 static void msg_close_conf(struct work_struct *work)
452 struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
453 struct rnbd_clt_dev *dev = iu->dev;
455 wake_up_iu_comp(iu, iu->errno);
456 rnbd_put_iu(dev->sess, iu);
457 rnbd_clt_put_dev(dev);
460 static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, bool wait)
462 struct rnbd_clt_session *sess = dev->sess;
463 struct rnbd_msg_close msg;
467 .iov_len = sizeof(msg)
471 iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
478 sg_mark_end(&iu->sglist[0]);
480 msg.hdr.type = cpu_to_le16(RNBD_MSG_CLOSE);
481 msg.device_id = cpu_to_le32(device_id);
483 WARN_ON(!rnbd_clt_get_dev(dev));
484 err = send_usr_msg(sess->rtrs, WRITE, iu, &vec, 0, NULL, 0,
485 msg_close_conf, &errno, wait);
487 rnbd_clt_put_dev(dev);
488 rnbd_put_iu(sess, iu);
493 rnbd_put_iu(sess, iu);
497 static void msg_open_conf(struct work_struct *work)
499 struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
500 struct rnbd_msg_open_rsp *rsp = iu->buf;
501 struct rnbd_clt_dev *dev = iu->dev;
502 int errno = iu->errno;
506 "Opening failed, server responded: %d\n",
509 errno = process_msg_open_rsp(dev, rsp);
511 u32 device_id = le32_to_cpu(rsp->device_id);
513 * If server thinks its fine, but we fail to process
514 * then be nice and send a close to server.
516 (void)send_msg_close(dev, device_id, NO_WAIT);
520 wake_up_iu_comp(iu, errno);
521 rnbd_put_iu(dev->sess, iu);
522 rnbd_clt_put_dev(dev);
525 static void msg_sess_info_conf(struct work_struct *work)
527 struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
528 struct rnbd_msg_sess_info_rsp *rsp = iu->buf;
529 struct rnbd_clt_session *sess = iu->sess;
532 sess->ver = min_t(u8, rsp->ver, RNBD_PROTO_VER_MAJOR);
535 wake_up_iu_comp(iu, iu->errno);
536 rnbd_put_iu(sess, iu);
537 rnbd_clt_put_sess(sess);
540 static int send_msg_open(struct rnbd_clt_dev *dev, bool wait)
542 struct rnbd_clt_session *sess = dev->sess;
543 struct rnbd_msg_open_rsp *rsp;
544 struct rnbd_msg_open msg;
548 .iov_len = sizeof(msg)
552 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
556 iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
565 sg_init_one(iu->sglist, rsp, sizeof(*rsp));
567 msg.hdr.type = cpu_to_le16(RNBD_MSG_OPEN);
568 msg.access_mode = dev->access_mode;
569 strlcpy(msg.dev_name, dev->pathname, sizeof(msg.dev_name));
571 WARN_ON(!rnbd_clt_get_dev(dev));
572 err = send_usr_msg(sess->rtrs, READ, iu,
573 &vec, sizeof(*rsp), iu->sglist, 1,
574 msg_open_conf, &errno, wait);
576 rnbd_clt_put_dev(dev);
577 rnbd_put_iu(sess, iu);
583 rnbd_put_iu(sess, iu);
587 static int send_msg_sess_info(struct rnbd_clt_session *sess, bool wait)
589 struct rnbd_msg_sess_info_rsp *rsp;
590 struct rnbd_msg_sess_info msg;
594 .iov_len = sizeof(msg)
598 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
602 iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
611 sg_init_one(iu->sglist, rsp, sizeof(*rsp));
613 msg.hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO);
614 msg.ver = RNBD_PROTO_VER_MAJOR;
616 if (!rnbd_clt_get_sess(sess)) {
618 * That can happen only in one case, when RTRS has restablished
619 * the connection and link_ev() is called, but session is almost
620 * dead, last reference on session is put and caller is waiting
621 * for RTRS to close everything.
626 err = send_usr_msg(sess->rtrs, READ, iu,
627 &vec, sizeof(*rsp), iu->sglist, 1,
628 msg_sess_info_conf, &errno, wait);
630 rnbd_clt_put_sess(sess);
632 rnbd_put_iu(sess, iu);
638 rnbd_put_iu(sess, iu);
642 static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess)
644 struct rnbd_clt_dev *dev;
646 mutex_lock(&sess->lock);
647 list_for_each_entry(dev, &sess->devs_list, list) {
648 rnbd_clt_err(dev, "Device disconnected.\n");
650 mutex_lock(&dev->lock);
651 if (dev->dev_state == DEV_STATE_MAPPED)
652 dev->dev_state = DEV_STATE_MAPPED_DISCONNECTED;
653 mutex_unlock(&dev->lock);
655 mutex_unlock(&sess->lock);
658 static void remap_devs(struct rnbd_clt_session *sess)
660 struct rnbd_clt_dev *dev;
661 struct rtrs_attrs attrs;
665 * Careful here: we are called from RTRS link event directly,
666 * thus we can't send any RTRS request and wait for response
667 * or RTRS will not be able to complete request with failure
668 * if something goes wrong (failing of outstanding requests
669 * happens exactly from the context where we are blocking now).
671 * So to avoid deadlocks each usr message sent from here must
675 err = send_msg_sess_info(sess, NO_WAIT);
677 pr_err("send_msg_sess_info(\"%s\"): %d\n", sess->sessname, err);
681 rtrs_clt_query(sess->rtrs, &attrs);
682 mutex_lock(&sess->lock);
683 sess->max_io_size = attrs.max_io_size;
685 list_for_each_entry(dev, &sess->devs_list, list) {
688 mutex_lock(&dev->lock);
689 skip = (dev->dev_state == DEV_STATE_INIT);
690 mutex_unlock(&dev->lock);
693 * When device is establishing connection for the first
694 * time - do not remap, it will be closed soon.
698 rnbd_clt_info(dev, "session reconnected, remapping device\n");
699 err = send_msg_open(dev, NO_WAIT);
701 rnbd_clt_err(dev, "send_msg_open(): %d\n", err);
705 mutex_unlock(&sess->lock);
708 static void rnbd_clt_link_ev(void *priv, enum rtrs_clt_link_ev ev)
710 struct rnbd_clt_session *sess = priv;
713 case RTRS_CLT_LINK_EV_DISCONNECTED:
714 set_dev_states_to_disconnected(sess);
716 case RTRS_CLT_LINK_EV_RECONNECTED:
720 pr_err("Unknown session event received (%d), session: %s\n",
725 static void rnbd_init_cpu_qlists(struct rnbd_cpu_qlist __percpu *cpu_queues)
728 struct rnbd_cpu_qlist *cpu_q;
730 for_each_possible_cpu(cpu) {
731 cpu_q = per_cpu_ptr(cpu_queues, cpu);
734 INIT_LIST_HEAD(&cpu_q->requeue_list);
735 spin_lock_init(&cpu_q->requeue_lock);
739 static void destroy_mq_tags(struct rnbd_clt_session *sess)
741 if (sess->tag_set.tags)
742 blk_mq_free_tag_set(&sess->tag_set);
745 static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess)
747 sess->rtrs_ready = true;
748 wake_up_all(&sess->rtrs_waitq);
751 static void close_rtrs(struct rnbd_clt_session *sess)
755 if (!IS_ERR_OR_NULL(sess->rtrs)) {
756 rtrs_clt_close(sess->rtrs);
758 wake_up_rtrs_waiters(sess);
762 static void free_sess(struct rnbd_clt_session *sess)
764 WARN_ON(!list_empty(&sess->devs_list));
769 destroy_mq_tags(sess);
770 if (!list_empty(&sess->list)) {
771 mutex_lock(&sess_lock);
772 list_del(&sess->list);
773 mutex_unlock(&sess_lock);
775 free_percpu(sess->cpu_queues);
776 free_percpu(sess->cpu_rr);
777 mutex_destroy(&sess->lock);
781 static struct rnbd_clt_session *alloc_sess(const char *sessname)
783 struct rnbd_clt_session *sess;
786 sess = kzalloc_node(sizeof(*sess), GFP_KERNEL, NUMA_NO_NODE);
788 return ERR_PTR(-ENOMEM);
789 strlcpy(sess->sessname, sessname, sizeof(sess->sessname));
790 atomic_set(&sess->busy, 0);
791 mutex_init(&sess->lock);
792 INIT_LIST_HEAD(&sess->devs_list);
793 INIT_LIST_HEAD(&sess->list);
794 bitmap_zero(sess->cpu_queues_bm, NR_CPUS);
795 init_waitqueue_head(&sess->rtrs_waitq);
796 refcount_set(&sess->refcount, 1);
798 sess->cpu_queues = alloc_percpu(struct rnbd_cpu_qlist);
799 if (!sess->cpu_queues) {
803 rnbd_init_cpu_qlists(sess->cpu_queues);
806 * That is simple percpu variable which stores cpu indeces, which are
807 * incremented on each access. We need that for the sake of fairness
808 * to wake up queues in a round-robin manner.
810 sess->cpu_rr = alloc_percpu(int);
815 for_each_possible_cpu(cpu)
816 * per_cpu_ptr(sess->cpu_rr, cpu) = cpu;
826 static int wait_for_rtrs_connection(struct rnbd_clt_session *sess)
828 wait_event(sess->rtrs_waitq, sess->rtrs_ready);
829 if (IS_ERR_OR_NULL(sess->rtrs))
835 static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess)
836 __releases(&sess_lock)
837 __acquires(&sess_lock)
841 prepare_to_wait(&sess->rtrs_waitq, &wait, TASK_UNINTERRUPTIBLE);
842 if (IS_ERR_OR_NULL(sess->rtrs)) {
843 finish_wait(&sess->rtrs_waitq, &wait);
846 mutex_unlock(&sess_lock);
847 /* loop in caller, see __find_and_get_sess().
848 * You can't leave mutex locked and call schedule(), you will catch a
849 * deadlock with a caller of free_sess(), which has just put the last
850 * reference and is about to take the sess_lock in order to delete
851 * the session from the list.
854 mutex_lock(&sess_lock);
857 static struct rnbd_clt_session *__find_and_get_sess(const char *sessname)
858 __releases(&sess_lock)
859 __acquires(&sess_lock)
861 struct rnbd_clt_session *sess, *sn;
865 list_for_each_entry_safe(sess, sn, &sess_list, list) {
866 if (strcmp(sessname, sess->sessname))
869 if (sess->rtrs_ready && IS_ERR_OR_NULL(sess->rtrs))
871 * No RTRS connection, session is dying.
875 if (rnbd_clt_get_sess(sess)) {
877 * Alive session is found, wait for RTRS connection.
879 mutex_unlock(&sess_lock);
880 err = wait_for_rtrs_connection(sess);
882 rnbd_clt_put_sess(sess);
883 mutex_lock(&sess_lock);
886 /* Session is dying, repeat the loop */
892 * Ref is 0, session is dying, wait for RTRS disconnect
893 * in order to avoid session names clashes.
895 wait_for_rtrs_disconnection(sess);
897 * RTRS is disconnected and soon session will be freed,
907 rnbd_clt_session *find_or_create_sess(const char *sessname, bool *first)
909 struct rnbd_clt_session *sess = NULL;
911 mutex_lock(&sess_lock);
912 sess = __find_and_get_sess(sessname);
914 sess = alloc_sess(sessname);
916 mutex_unlock(&sess_lock);
919 list_add(&sess->list, &sess_list);
923 mutex_unlock(&sess_lock);
928 static int rnbd_client_open(struct block_device *block_device, fmode_t mode)
930 struct rnbd_clt_dev *dev = block_device->bd_disk->private_data;
932 if (dev->read_only && (mode & FMODE_WRITE))
935 if (dev->dev_state == DEV_STATE_UNMAPPED ||
936 !rnbd_clt_get_dev(dev))
942 static void rnbd_client_release(struct gendisk *gen, fmode_t mode)
944 struct rnbd_clt_dev *dev = gen->private_data;
946 rnbd_clt_put_dev(dev);
949 static int rnbd_client_getgeo(struct block_device *block_device,
950 struct hd_geometry *geo)
953 struct rnbd_clt_dev *dev;
955 dev = block_device->bd_disk->private_data;
956 size = dev->size * (dev->logical_block_size / SECTOR_SIZE);
957 geo->cylinders = size >> 6; /* size/64 */
965 static const struct block_device_operations rnbd_client_ops = {
966 .owner = THIS_MODULE,
967 .open = rnbd_client_open,
968 .release = rnbd_client_release,
969 .getgeo = rnbd_client_getgeo
972 /* The amount of data that belongs to an I/O and the amount of data that
973 * should be read or written to the disk (bi_size) can differ.
975 * E.g. When WRITE_SAME is used, only a small amount of data is
976 * transferred that is then written repeatedly over a lot of sectors.
978 * Get the size of data to be transferred via RTRS by summing up the size
979 * of the scather-gather list entries.
981 static size_t rnbd_clt_get_sg_size(struct scatterlist *sglist, u32 len)
983 struct scatterlist *sg;
987 for_each_sg(sglist, sg, len, i)
992 static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev,
996 struct rtrs_clt *rtrs = dev->sess->rtrs;
997 struct rtrs_permit *permit = iu->permit;
998 struct rnbd_msg_io msg;
999 struct rtrs_clt_req_ops req_ops;
1000 unsigned int sg_cnt = 0;
1007 msg.sector = cpu_to_le64(blk_rq_pos(rq));
1008 msg.bi_size = cpu_to_le32(blk_rq_bytes(rq));
1009 msg.rw = cpu_to_le32(rq_to_rnbd_flags(rq));
1010 msg.prio = cpu_to_le16(req_get_ioprio(rq));
1013 * We only support discards with single segment for now.
1016 if (req_op(rq) != REQ_OP_DISCARD)
1017 sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sglist);
1020 /* Do not forget to mark the end */
1021 sg_mark_end(&iu->sglist[0]);
1023 msg.hdr.type = cpu_to_le16(RNBD_MSG_IO);
1024 msg.device_id = cpu_to_le32(dev->device_id);
1026 vec = (struct kvec) {
1028 .iov_len = sizeof(msg)
1030 size = rnbd_clt_get_sg_size(iu->sglist, sg_cnt);
1031 req_ops = (struct rtrs_clt_req_ops) {
1033 .conf_fn = msg_io_conf,
1035 err = rtrs_clt_request(rq_data_dir(rq), &req_ops, rtrs, permit,
1036 &vec, 1, size, iu->sglist, sg_cnt);
1037 if (unlikely(err)) {
1038 rnbd_clt_err_rl(dev, "RTRS failed to transfer IO, err: %d\n",
1047 * rnbd_clt_dev_add_to_requeue() - add device to requeue if session is busy
1048 * @dev: Device to be checked
1049 * @q: Queue to be added to the requeue list if required
1052 * If session is busy, that means someone will requeue us when resources
1053 * are freed. If session is not doing anything - device is not added to
1054 * the list and @false is returned.
1056 static bool rnbd_clt_dev_add_to_requeue(struct rnbd_clt_dev *dev,
1057 struct rnbd_queue *q)
1059 struct rnbd_clt_session *sess = dev->sess;
1060 struct rnbd_cpu_qlist *cpu_q;
1061 unsigned long flags;
1065 cpu_q = get_cpu_ptr(sess->cpu_queues);
1066 spin_lock_irqsave(&cpu_q->requeue_lock, flags);
1068 if (likely(!test_and_set_bit_lock(0, &q->in_list))) {
1069 if (WARN_ON(!list_empty(&q->requeue_list)))
1072 need_set = !test_bit(cpu_q->cpu, sess->cpu_queues_bm);
1074 set_bit(cpu_q->cpu, sess->cpu_queues_bm);
1075 /* Paired with rnbd_put_permit(). Set a bit first
1076 * and then observe the busy counter.
1078 smp_mb__before_atomic();
1080 if (likely(atomic_read(&sess->busy))) {
1081 list_add_tail(&q->requeue_list, &cpu_q->requeue_list);
1083 /* Very unlikely, but possible: busy counter was
1084 * observed as zero. Drop all bits and return
1085 * false to restart the queue by ourselves.
1088 clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
1089 clear_bit_unlock(0, &q->in_list);
1094 spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
1095 put_cpu_ptr(sess->cpu_queues);
1100 static void rnbd_clt_dev_kick_mq_queue(struct rnbd_clt_dev *dev,
1101 struct blk_mq_hw_ctx *hctx,
1104 struct rnbd_queue *q = hctx->driver_data;
1106 if (delay != RNBD_DELAY_IFBUSY)
1107 blk_mq_delay_run_hw_queue(hctx, delay);
1108 else if (unlikely(!rnbd_clt_dev_add_to_requeue(dev, q)))
1110 * If session is not busy we have to restart
1111 * the queue ourselves.
1113 blk_mq_delay_run_hw_queue(hctx, 10/*ms*/);
1116 static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
1117 const struct blk_mq_queue_data *bd)
1119 struct request *rq = bd->rq;
1120 struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
1121 struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
1124 if (unlikely(dev->dev_state != DEV_STATE_MAPPED))
1125 return BLK_STS_IOERR;
1127 iu->permit = rnbd_get_permit(dev->sess, RTRS_IO_CON,
1128 RTRS_PERMIT_NOWAIT);
1129 if (unlikely(!iu->permit)) {
1130 rnbd_clt_dev_kick_mq_queue(dev, hctx, RNBD_DELAY_IFBUSY);
1131 return BLK_STS_RESOURCE;
1134 blk_mq_start_request(rq);
1135 err = rnbd_client_xfer_request(dev, rq, iu);
1136 if (likely(err == 0))
1138 if (unlikely(err == -EAGAIN || err == -ENOMEM)) {
1139 rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
1140 rnbd_put_permit(dev->sess, iu->permit);
1141 return BLK_STS_RESOURCE;
1144 rnbd_put_permit(dev->sess, iu->permit);
1145 return BLK_STS_IOERR;
1148 static int rnbd_init_request(struct blk_mq_tag_set *set, struct request *rq,
1149 unsigned int hctx_idx, unsigned int numa_node)
1151 struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
1153 sg_init_table(iu->sglist, BMAX_SEGMENTS);
1157 static struct blk_mq_ops rnbd_mq_ops = {
1158 .queue_rq = rnbd_queue_rq,
1159 .init_request = rnbd_init_request,
1160 .complete = rnbd_softirq_done_fn,
1163 static int setup_mq_tags(struct rnbd_clt_session *sess)
1165 struct blk_mq_tag_set *tag_set = &sess->tag_set;
1167 memset(tag_set, 0, sizeof(*tag_set));
1168 tag_set->ops = &rnbd_mq_ops;
1169 tag_set->queue_depth = sess->queue_depth;
1170 tag_set->numa_node = NUMA_NO_NODE;
1171 tag_set->flags = BLK_MQ_F_SHOULD_MERGE |
1172 BLK_MQ_F_TAG_QUEUE_SHARED;
1173 tag_set->cmd_size = sizeof(struct rnbd_iu);
1174 tag_set->nr_hw_queues = num_online_cpus();
1176 return blk_mq_alloc_tag_set(tag_set);
1179 static struct rnbd_clt_session *
1180 find_and_get_or_create_sess(const char *sessname,
1181 const struct rtrs_addr *paths,
1182 size_t path_cnt, u16 port_nr)
1184 struct rnbd_clt_session *sess;
1185 struct rtrs_attrs attrs;
1188 struct rtrs_clt_ops rtrs_ops;
1190 sess = find_or_create_sess(sessname, &first);
1191 if (sess == ERR_PTR(-ENOMEM))
1192 return ERR_PTR(-ENOMEM);
1196 rtrs_ops = (struct rtrs_clt_ops) {
1198 .link_ev = rnbd_clt_link_ev,
1201 * Nothing was found, establish rtrs connection and proceed further.
1203 sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,
1204 paths, path_cnt, port_nr,
1205 sizeof(struct rnbd_iu),
1206 RECONNECT_DELAY, BMAX_SEGMENTS,
1207 BLK_MAX_SEGMENT_SIZE,
1209 if (IS_ERR(sess->rtrs)) {
1210 err = PTR_ERR(sess->rtrs);
1211 goto wake_up_and_put;
1213 rtrs_clt_query(sess->rtrs, &attrs);
1214 sess->max_io_size = attrs.max_io_size;
1215 sess->queue_depth = attrs.queue_depth;
1217 err = setup_mq_tags(sess);
1221 err = send_msg_sess_info(sess, WAIT);
1225 wake_up_rtrs_waiters(sess);
1232 rnbd_clt_put_sess(sess);
1234 return ERR_PTR(err);
1237 wake_up_rtrs_waiters(sess);
1241 static inline void rnbd_init_hw_queue(struct rnbd_clt_dev *dev,
1242 struct rnbd_queue *q,
1243 struct blk_mq_hw_ctx *hctx)
1245 INIT_LIST_HEAD(&q->requeue_list);
1250 static void rnbd_init_mq_hw_queues(struct rnbd_clt_dev *dev)
1253 struct blk_mq_hw_ctx *hctx;
1254 struct rnbd_queue *q;
1256 queue_for_each_hw_ctx(dev->queue, hctx, i) {
1257 q = &dev->hw_queues[i];
1258 rnbd_init_hw_queue(dev, q, hctx);
1259 hctx->driver_data = q;
1263 static int setup_mq_dev(struct rnbd_clt_dev *dev)
1265 dev->queue = blk_mq_init_queue(&dev->sess->tag_set);
1266 if (IS_ERR(dev->queue)) {
1267 rnbd_clt_err(dev, "Initializing multiqueue queue failed, err: %ld\n",
1268 PTR_ERR(dev->queue));
1269 return PTR_ERR(dev->queue);
1271 rnbd_init_mq_hw_queues(dev);
1275 static void setup_request_queue(struct rnbd_clt_dev *dev)
1277 blk_queue_logical_block_size(dev->queue, dev->logical_block_size);
1278 blk_queue_physical_block_size(dev->queue, dev->physical_block_size);
1279 blk_queue_max_hw_sectors(dev->queue, dev->max_hw_sectors);
1280 blk_queue_max_write_same_sectors(dev->queue,
1281 dev->max_write_same_sectors);
1284 * we don't support discards to "discontiguous" segments
1287 blk_queue_max_discard_segments(dev->queue, 1);
1289 blk_queue_max_discard_sectors(dev->queue, dev->max_discard_sectors);
1290 dev->queue->limits.discard_granularity = dev->discard_granularity;
1291 dev->queue->limits.discard_alignment = dev->discard_alignment;
1292 if (dev->max_discard_sectors)
1293 blk_queue_flag_set(QUEUE_FLAG_DISCARD, dev->queue);
1294 if (dev->secure_discard)
1295 blk_queue_flag_set(QUEUE_FLAG_SECERASE, dev->queue);
1297 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, dev->queue);
1298 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, dev->queue);
1299 blk_queue_max_segments(dev->queue, dev->max_segments);
1300 blk_queue_io_opt(dev->queue, dev->sess->max_io_size);
1301 blk_queue_virt_boundary(dev->queue, SZ_4K - 1);
1302 blk_queue_write_cache(dev->queue, true, true);
1303 dev->queue->queuedata = dev;
1306 static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx)
1308 dev->gd->major = rnbd_client_major;
1309 dev->gd->first_minor = idx << RNBD_PART_BITS;
1310 dev->gd->fops = &rnbd_client_ops;
1311 dev->gd->queue = dev->queue;
1312 dev->gd->private_data = dev;
1313 snprintf(dev->gd->disk_name, sizeof(dev->gd->disk_name), "rnbd%d",
1315 pr_debug("disk_name=%s, capacity=%zu\n",
1317 dev->nsectors * (dev->logical_block_size / SECTOR_SIZE)
1320 set_capacity(dev->gd, dev->nsectors);
1322 if (dev->access_mode == RNBD_ACCESS_RO) {
1323 dev->read_only = true;
1324 set_disk_ro(dev->gd, true);
1326 dev->read_only = false;
1329 if (!dev->rotational)
1330 blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue);
1333 static int rnbd_client_setup_device(struct rnbd_clt_session *sess,
1334 struct rnbd_clt_dev *dev, int idx)
1338 dev->size = dev->nsectors * dev->logical_block_size;
1340 err = setup_mq_dev(dev);
1344 setup_request_queue(dev);
1346 dev->gd = alloc_disk_node(1 << RNBD_PART_BITS, NUMA_NO_NODE);
1348 blk_cleanup_queue(dev->queue);
1352 rnbd_clt_setup_gen_disk(dev, idx);
1357 static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
1358 enum rnbd_access_mode access_mode,
1359 const char *pathname)
1361 struct rnbd_clt_dev *dev;
1364 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, NUMA_NO_NODE);
1366 return ERR_PTR(-ENOMEM);
1368 dev->hw_queues = kcalloc(nr_cpu_ids, sizeof(*dev->hw_queues),
1370 if (!dev->hw_queues) {
1375 mutex_lock(&ida_lock);
1376 ret = ida_simple_get(&index_ida, 0, 1 << (MINORBITS - RNBD_PART_BITS),
1378 mutex_unlock(&ida_lock);
1380 pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n",
1381 pathname, sess->sessname, ret);
1384 dev->clt_device_id = ret;
1386 dev->access_mode = access_mode;
1387 strlcpy(dev->pathname, pathname, sizeof(dev->pathname));
1388 mutex_init(&dev->lock);
1389 refcount_set(&dev->refcount, 1);
1390 dev->dev_state = DEV_STATE_INIT;
1393 * Here we called from sysfs entry, thus clt-sysfs is
1394 * responsible that session will not disappear.
1396 WARN_ON(!rnbd_clt_get_sess(sess));
1401 kfree(dev->hw_queues);
1404 return ERR_PTR(ret);
1407 static bool __exists_dev(const char *pathname)
1409 struct rnbd_clt_session *sess;
1410 struct rnbd_clt_dev *dev;
1413 list_for_each_entry(sess, &sess_list, list) {
1414 mutex_lock(&sess->lock);
1415 list_for_each_entry(dev, &sess->devs_list, list) {
1416 if (!strncmp(dev->pathname, pathname,
1417 sizeof(dev->pathname))) {
1422 mutex_unlock(&sess->lock);
1430 static bool exists_devpath(const char *pathname)
1434 mutex_lock(&sess_lock);
1435 found = __exists_dev(pathname);
1436 mutex_unlock(&sess_lock);
1441 static bool insert_dev_if_not_exists_devpath(const char *pathname,
1442 struct rnbd_clt_session *sess,
1443 struct rnbd_clt_dev *dev)
1447 mutex_lock(&sess_lock);
1448 found = __exists_dev(pathname);
1450 mutex_lock(&sess->lock);
1451 list_add_tail(&dev->list, &sess->devs_list);
1452 mutex_unlock(&sess->lock);
1454 mutex_unlock(&sess_lock);
1459 static void delete_dev(struct rnbd_clt_dev *dev)
1461 struct rnbd_clt_session *sess = dev->sess;
1463 mutex_lock(&sess->lock);
1464 list_del(&dev->list);
1465 mutex_unlock(&sess->lock);
1468 struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname,
1469 struct rtrs_addr *paths,
1470 size_t path_cnt, u16 port_nr,
1471 const char *pathname,
1472 enum rnbd_access_mode access_mode)
1474 struct rnbd_clt_session *sess;
1475 struct rnbd_clt_dev *dev;
1478 if (exists_devpath(pathname))
1479 return ERR_PTR(-EEXIST);
1481 sess = find_and_get_or_create_sess(sessname, paths, path_cnt, port_nr);
1483 return ERR_CAST(sess);
1485 dev = init_dev(sess, access_mode, pathname);
1487 pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %ld\n",
1488 pathname, sess->sessname, PTR_ERR(dev));
1492 if (insert_dev_if_not_exists_devpath(pathname, sess, dev)) {
1496 ret = send_msg_open(dev, WAIT);
1499 "map_device: failed, can't open remote device, err: %d\n",
1503 mutex_lock(&dev->lock);
1504 pr_debug("Opened remote device: session=%s, path='%s'\n",
1505 sess->sessname, pathname);
1506 ret = rnbd_client_setup_device(sess, dev, dev->clt_device_id);
1509 "map_device: Failed to configure device, err: %d\n",
1511 mutex_unlock(&dev->lock);
1516 "map_device: Device mapped as %s (nsectors: %zu, logical_block_size: %d, physical_block_size: %d, max_write_same_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, rotational: %d)\n",
1517 dev->gd->disk_name, dev->nsectors,
1518 dev->logical_block_size, dev->physical_block_size,
1519 dev->max_write_same_sectors, dev->max_discard_sectors,
1520 dev->discard_granularity, dev->discard_alignment,
1521 dev->secure_discard, dev->max_segments,
1522 dev->max_hw_sectors, dev->rotational);
1524 mutex_unlock(&dev->lock);
1527 rnbd_clt_put_sess(sess);
1532 send_msg_close(dev, dev->device_id, WAIT);
1536 rnbd_clt_put_dev(dev);
1538 rnbd_clt_put_sess(sess);
1540 return ERR_PTR(ret);
1543 static void destroy_gen_disk(struct rnbd_clt_dev *dev)
1545 del_gendisk(dev->gd);
1546 blk_cleanup_queue(dev->queue);
1550 static void destroy_sysfs(struct rnbd_clt_dev *dev,
1551 const struct attribute *sysfs_self)
1553 rnbd_clt_remove_dev_symlink(dev);
1554 if (dev->kobj.state_initialized) {
1556 /* To avoid deadlock firstly remove itself */
1557 sysfs_remove_file_self(&dev->kobj, sysfs_self);
1558 kobject_del(&dev->kobj);
1559 kobject_put(&dev->kobj);
1563 int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
1564 const struct attribute *sysfs_self)
1566 struct rnbd_clt_session *sess = dev->sess;
1567 int refcount, ret = 0;
1570 mutex_lock(&dev->lock);
1571 if (dev->dev_state == DEV_STATE_UNMAPPED) {
1572 rnbd_clt_info(dev, "Device is already being unmapped\n");
1576 refcount = refcount_read(&dev->refcount);
1577 if (!force && refcount > 1) {
1579 "Closing device failed, device is in use, (%d device users)\n",
1584 was_mapped = (dev->dev_state == DEV_STATE_MAPPED);
1585 dev->dev_state = DEV_STATE_UNMAPPED;
1586 mutex_unlock(&dev->lock);
1589 destroy_sysfs(dev, sysfs_self);
1590 destroy_gen_disk(dev);
1591 if (was_mapped && sess->rtrs)
1592 send_msg_close(dev, dev->device_id, WAIT);
1594 rnbd_clt_info(dev, "Device is unmapped\n");
1596 /* Likely last reference put */
1597 rnbd_clt_put_dev(dev);
1600 * Here device and session can be vanished!
1605 mutex_unlock(&dev->lock);
1610 int rnbd_clt_remap_device(struct rnbd_clt_dev *dev)
1614 mutex_lock(&dev->lock);
1615 if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED)
1617 else if (dev->dev_state == DEV_STATE_UNMAPPED)
1619 else if (dev->dev_state == DEV_STATE_MAPPED)
1623 mutex_unlock(&dev->lock);
1625 rnbd_clt_info(dev, "Remapping device.\n");
1626 err = send_msg_open(dev, WAIT);
1628 rnbd_clt_err(dev, "remap_device: %d\n", err);
1634 static void unmap_device_work(struct work_struct *work)
1636 struct rnbd_clt_dev *dev;
1638 dev = container_of(work, typeof(*dev), unmap_on_rmmod_work);
1639 rnbd_clt_unmap_device(dev, true, NULL);
1642 static void rnbd_destroy_sessions(void)
1644 struct rnbd_clt_session *sess, *sn;
1645 struct rnbd_clt_dev *dev, *tn;
1647 /* Firstly forbid access through sysfs interface */
1648 rnbd_clt_destroy_default_group();
1649 rnbd_clt_destroy_sysfs_files();
1652 * Here at this point there is no any concurrent access to sessions
1653 * list and devices list:
1654 * 1. New session or device can'be be created - session sysfs files
1656 * 2. Device or session can't be removed - module reference is taken
1657 * into account in unmap device sysfs callback.
1658 * 3. No IO requests inflight - each file open of block_dev increases
1659 * module reference in get_disk().
1661 * But still there can be user requests inflights, which are sent by
1662 * asynchronous send_msg_*() functions, thus before unmapping devices
1663 * RTRS session must be explicitly closed.
1666 list_for_each_entry_safe(sess, sn, &sess_list, list) {
1667 WARN_ON(!rnbd_clt_get_sess(sess));
1669 list_for_each_entry_safe(dev, tn, &sess->devs_list, list) {
1671 * Here unmap happens in parallel for only one reason:
1672 * blk_cleanup_queue() takes around half a second, so
1673 * on huge amount of devices the whole module unload
1674 * procedure takes minutes.
1676 INIT_WORK(&dev->unmap_on_rmmod_work, unmap_device_work);
1677 queue_work(system_long_wq, &dev->unmap_on_rmmod_work);
1679 rnbd_clt_put_sess(sess);
1681 /* Wait for all scheduled unmap works */
1682 flush_workqueue(system_long_wq);
1683 WARN_ON(!list_empty(&sess_list));
1686 static int __init rnbd_client_init(void)
1690 BUILD_BUG_ON(sizeof(struct rnbd_msg_hdr) != 4);
1691 BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info) != 36);
1692 BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info_rsp) != 36);
1693 BUILD_BUG_ON(sizeof(struct rnbd_msg_open) != 264);
1694 BUILD_BUG_ON(sizeof(struct rnbd_msg_close) != 8);
1695 BUILD_BUG_ON(sizeof(struct rnbd_msg_open_rsp) != 56);
1696 rnbd_client_major = register_blkdev(rnbd_client_major, "rnbd");
1697 if (rnbd_client_major <= 0) {
1698 pr_err("Failed to load module, block device registration failed\n");
1702 err = rnbd_clt_create_sysfs_files();
1704 pr_err("Failed to load module, creating sysfs device files failed, err: %d\n",
1706 unregister_blkdev(rnbd_client_major, "rnbd");
1712 static void __exit rnbd_client_exit(void)
1714 rnbd_destroy_sessions();
1715 unregister_blkdev(rnbd_client_major, "rnbd");
1716 ida_destroy(&index_ida);
1719 module_init(rnbd_client_init);
1720 module_exit(rnbd_client_exit);