drivers/block/null_blk_main.c

   1 /*
   2  * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
   3  * Shaohua Li <shli@fb.com>
   4  */
   5 #include <linux/module.h>
   6
   7 #include <linux/moduleparam.h>
   8 #include <linux/sched.h>
   9 #include <linux/fs.h>
  10 #include <linux/init.h>
  11 #include "null_blk.h"
  12
  13 #define PAGE_SECTORS_SHIFT      (PAGE_SHIFT - SECTOR_SHIFT)
  14 #define PAGE_SECTORS            (1 << PAGE_SECTORS_SHIFT)
  15 #define SECTOR_MASK             (PAGE_SECTORS - 1)
  16
  17 #define FREE_BATCH              16
  18
  19 #define TICKS_PER_SEC           50ULL
  20 #define TIMER_INTERVAL          (NSEC_PER_SEC / TICKS_PER_SEC)
  21
  22 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  23 static DECLARE_FAULT_ATTR(null_timeout_attr);
  24 static DECLARE_FAULT_ATTR(null_requeue_attr);
  25 #endif
  26
  27 static inline u64 mb_per_tick(int mbps)
  28 {
  29         return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
  30 }
  31
  32 /*
  33  * Status flags for nullb_device.
  34  *
  35  * CONFIGURED:  Device has been configured and turned on. Cannot reconfigure.
  36  * UP:          Device is currently on and visible in userspace.
  37  * THROTTLED:   Device is being throttled.
  38  * CACHE:       Device is using a write-back cache.
  39  */
  40 enum nullb_device_flags {
  41         NULLB_DEV_FL_CONFIGURED = 0,
  42         NULLB_DEV_FL_UP         = 1,
  43         NULLB_DEV_FL_THROTTLED  = 2,
  44         NULLB_DEV_FL_CACHE      = 3,
  45 };
  46
  47 #define MAP_SZ          ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
  48 /*
  49  * nullb_page is a page in memory for nullb devices.
  50  *
  51  * @page:       The page holding the data.
  52  * @bitmap:     The bitmap represents which sector in the page has data.
  53  *              Each bit represents one block size. For example, sector 8
  54  *              will use the 7th bit
  55  * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
  56  * page is being flushing to storage. FREE means the cache page is freed and
  57  * should be skipped from flushing to storage. Please see
  58  * null_make_cache_space
  59  */
  60 struct nullb_page {
  61         struct page *page;
  62         DECLARE_BITMAP(bitmap, MAP_SZ);
  63 };
  64 #define NULLB_PAGE_LOCK (MAP_SZ - 1)
  65 #define NULLB_PAGE_FREE (MAP_SZ - 2)
  66
  67 static LIST_HEAD(nullb_list);
  68 static struct mutex lock;
  69 static int null_major;
  70 static DEFINE_IDA(nullb_indexes);
  71 static struct blk_mq_tag_set tag_set;
  72
  73 enum {
  74         NULL_IRQ_NONE           = 0,
  75         NULL_IRQ_SOFTIRQ        = 1,
  76         NULL_IRQ_TIMER          = 2,
  77 };
  78
  79 enum {
  80         NULL_Q_BIO              = 0,
  81         NULL_Q_RQ               = 1,
  82         NULL_Q_MQ               = 2,
  83 };
  84
  85 static int g_no_sched;
  86 module_param_named(no_sched, g_no_sched, int, 0444);
  87 MODULE_PARM_DESC(no_sched, "No io scheduler");
  88
  89 static int g_submit_queues = 1;
  90 module_param_named(submit_queues, g_submit_queues, int, 0444);
  91 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  92
  93 static int g_home_node = NUMA_NO_NODE;
  94 module_param_named(home_node, g_home_node, int, 0444);
  95 MODULE_PARM_DESC(home_node, "Home node for the device");
  96
  97 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  98 static char g_timeout_str[80];
  99 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
 100
 101 static char g_requeue_str[80];
 102 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
 103 #endif
 104
 105 static int g_queue_mode = NULL_Q_MQ;
 106
 107 static int null_param_store_val(const char *str, int *val, int min, int max)
 108 {
 109         int ret, new_val;
 110
 111         ret = kstrtoint(str, 10, &new_val);
 112         if (ret)
 113                 return -EINVAL;
 114
 115         if (new_val < min || new_val > max)
 116                 return -EINVAL;
 117
 118         *val = new_val;
 119         return 0;
 120 }
 121
 122 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
 123 {
 124         return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
 125 }
 126
 127 static const struct kernel_param_ops null_queue_mode_param_ops = {
 128         .set    = null_set_queue_mode,
 129         .get    = param_get_int,
 130 };
 131
 132 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
 133 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 134
 135 static int g_gb = 250;
 136 module_param_named(gb, g_gb, int, 0444);
 137 MODULE_PARM_DESC(gb, "Size in GB");
 138
 139 static int g_bs = 512;
 140 module_param_named(bs, g_bs, int, 0444);
 141 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 142
 143 static int nr_devices = 1;
 144 module_param(nr_devices, int, 0444);
 145 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 146
 147 static bool g_blocking;
 148 module_param_named(blocking, g_blocking, bool, 0444);
 149 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 150
 151 static bool shared_tags;
 152 module_param(shared_tags, bool, 0444);
 153 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 154
 155 static int g_irqmode = NULL_IRQ_SOFTIRQ;
 156
 157 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 158 {
 159         return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
 160                                         NULL_IRQ_TIMER);
 161 }
 162
 163 static const struct kernel_param_ops null_irqmode_param_ops = {
 164         .set    = null_set_irqmode,
 165         .get    = param_get_int,
 166 };
 167
 168 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
 169 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 170
 171 static unsigned long g_completion_nsec = 10000;
 172 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
 173 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 174
 175 static int g_hw_queue_depth = 64;
 176 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
 177 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 178
 179 static bool g_use_per_node_hctx;
 180 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
 181 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 182
 183 static bool g_zoned;
 184 module_param_named(zoned, g_zoned, bool, S_IRUGO);
 185 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
 186
 187 static unsigned long g_zone_size = 256;
 188 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
 189 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
 190
 191 static unsigned int g_zone_nr_conv;
 192 module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
 193 MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
 194
 195 static struct nullb_device *null_alloc_dev(void);
 196 static void null_free_dev(struct nullb_device *dev);
 197 static void null_del_dev(struct nullb *nullb);
 198 static int null_add_dev(struct nullb_device *dev);
 199 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
 200
 201 static inline struct nullb_device *to_nullb_device(struct config_item *item)
 202 {
 203         return item ? container_of(item, struct nullb_device, item) : NULL;
 204 }
 205
 206 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
 207 {
 208         return snprintf(page, PAGE_SIZE, "%u\n", val);
 209 }
 210
 211 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
 212         char *page)
 213 {
 214         return snprintf(page, PAGE_SIZE, "%lu\n", val);
 215 }
 216
 217 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
 218 {
 219         return snprintf(page, PAGE_SIZE, "%u\n", val);
 220 }
 221
 222 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
 223         const char *page, size_t count)
 224 {
 225         unsigned int tmp;
 226         int result;
 227
 228         result = kstrtouint(page, 0, &tmp);
 229         if (result)
 230                 return result;
 231
 232         *val = tmp;
 233         return count;
 234 }
 235
 236 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
 237         const char *page, size_t count)
 238 {
 239         int result;
 240         unsigned long tmp;
 241
 242         result = kstrtoul(page, 0, &tmp);
 243         if (result)
 244                 return result;
 245
 246         *val = tmp;
 247         return count;
 248 }
 249
 250 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
 251         size_t count)
 252 {
 253         bool tmp;
 254         int result;
 255
 256         result = kstrtobool(page,  &tmp);
 257         if (result)
 258                 return result;
 259
 260         *val = tmp;
 261         return count;
 262 }
 263
 264 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
 265 #define NULLB_DEVICE_ATTR(NAME, TYPE)                                           \
 266 static ssize_t                                                                  \
 267 nullb_device_##NAME##_show(struct config_item *item, char *page)                \
 268 {                                                                               \
 269         return nullb_device_##TYPE##_attr_show(                                 \
 270                                 to_nullb_device(item)->NAME, page);             \
 271 }                                                                               \
 272 static ssize_t                                                                  \
 273 nullb_device_##NAME##_store(struct config_item *item, const char *page,         \
 274                             size_t count)                                       \
 275 {                                                                               \
 276         if (test_bit(NULLB_DEV_FL_CONFIGURED, &to_nullb_device(item)->flags))   \
 277                 return -EBUSY;                                                  \
 278         return nullb_device_##TYPE##_attr_store(                                \
 279                         &to_nullb_device(item)->NAME, page, count);             \
 280 }                                                                               \
 281 CONFIGFS_ATTR(nullb_device_, NAME);
 282
 283 NULLB_DEVICE_ATTR(size, ulong);
 284 NULLB_DEVICE_ATTR(completion_nsec, ulong);
 285 NULLB_DEVICE_ATTR(submit_queues, uint);
 286 NULLB_DEVICE_ATTR(home_node, uint);
 287 NULLB_DEVICE_ATTR(queue_mode, uint);
 288 NULLB_DEVICE_ATTR(blocksize, uint);
 289 NULLB_DEVICE_ATTR(irqmode, uint);
 290 NULLB_DEVICE_ATTR(hw_queue_depth, uint);
 291 NULLB_DEVICE_ATTR(index, uint);
 292 NULLB_DEVICE_ATTR(blocking, bool);
 293 NULLB_DEVICE_ATTR(use_per_node_hctx, bool);
 294 NULLB_DEVICE_ATTR(memory_backed, bool);
 295 NULLB_DEVICE_ATTR(discard, bool);
 296 NULLB_DEVICE_ATTR(mbps, uint);
 297 NULLB_DEVICE_ATTR(cache_size, ulong);
 298 NULLB_DEVICE_ATTR(zoned, bool);
 299 NULLB_DEVICE_ATTR(zone_size, ulong);
 300 NULLB_DEVICE_ATTR(zone_nr_conv, uint);
 301
 302 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
 303 {
 304         return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
 305 }
 306
 307 static ssize_t nullb_device_power_store(struct config_item *item,
 308                                      const char *page, size_t count)
 309 {
 310         struct nullb_device *dev = to_nullb_device(item);
 311         bool newp = false;
 312         ssize_t ret;
 313
 314         ret = nullb_device_bool_attr_store(&newp, page, count);
 315         if (ret < 0)
 316                 return ret;
 317
 318         if (!dev->power && newp) {
 319                 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
 320                         return count;
 321                 if (null_add_dev(dev)) {
 322                         clear_bit(NULLB_DEV_FL_UP, &dev->flags);
 323                         return -ENOMEM;
 324                 }
 325
 326                 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 327                 dev->power = newp;
 328         } else if (dev->power && !newp) {
 329                 mutex_lock(&lock);
 330                 dev->power = newp;
 331                 null_del_dev(dev->nullb);
 332                 mutex_unlock(&lock);
 333                 clear_bit(NULLB_DEV_FL_UP, &dev->flags);
 334                 clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 335         }
 336
 337         return count;
 338 }
 339
 340 CONFIGFS_ATTR(nullb_device_, power);
 341
 342 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
 343 {
 344         struct nullb_device *t_dev = to_nullb_device(item);
 345
 346         return badblocks_show(&t_dev->badblocks, page, 0);
 347 }
 348
 349 static ssize_t nullb_device_badblocks_store(struct config_item *item,
 350                                      const char *page, size_t count)
 351 {
 352         struct nullb_device *t_dev = to_nullb_device(item);
 353         char *orig, *buf, *tmp;
 354         u64 start, end;
 355         int ret;
 356
 357         orig = kstrndup(page, count, GFP_KERNEL);
 358         if (!orig)
 359                 return -ENOMEM;
 360
 361         buf = strstrip(orig);
 362
 363         ret = -EINVAL;
 364         if (buf[0] != '+' && buf[0] != '-')
 365                 goto out;
 366         tmp = strchr(&buf[1], '-');
 367         if (!tmp)
 368                 goto out;
 369         *tmp = '\0';
 370         ret = kstrtoull(buf + 1, 0, &start);
 371         if (ret)
 372                 goto out;
 373         ret = kstrtoull(tmp + 1, 0, &end);
 374         if (ret)
 375                 goto out;
 376         ret = -EINVAL;
 377         if (start > end)
 378                 goto out;
 379         /* enable badblocks */
 380         cmpxchg(&t_dev->badblocks.shift, -1, 0);
 381         if (buf[0] == '+')
 382                 ret = badblocks_set(&t_dev->badblocks, start,
 383                         end - start + 1, 1);
 384         else
 385                 ret = badblocks_clear(&t_dev->badblocks, start,
 386                         end - start + 1);
 387         if (ret == 0)
 388                 ret = count;
 389 out:
 390         kfree(orig);
 391         return ret;
 392 }
 393 CONFIGFS_ATTR(nullb_device_, badblocks);
 394
 395 static struct configfs_attribute *nullb_device_attrs[] = {
 396         &nullb_device_attr_size,
 397         &nullb_device_attr_completion_nsec,
 398         &nullb_device_attr_submit_queues,
 399         &nullb_device_attr_home_node,
 400         &nullb_device_attr_queue_mode,
 401         &nullb_device_attr_blocksize,
 402         &nullb_device_attr_irqmode,
 403         &nullb_device_attr_hw_queue_depth,
 404         &nullb_device_attr_index,
 405         &nullb_device_attr_blocking,
 406         &nullb_device_attr_use_per_node_hctx,
 407         &nullb_device_attr_power,
 408         &nullb_device_attr_memory_backed,
 409         &nullb_device_attr_discard,
 410         &nullb_device_attr_mbps,
 411         &nullb_device_attr_cache_size,
 412         &nullb_device_attr_badblocks,
 413         &nullb_device_attr_zoned,
 414         &nullb_device_attr_zone_size,
 415         &nullb_device_attr_zone_nr_conv,
 416         NULL,
 417 };
 418
 419 static void nullb_device_release(struct config_item *item)
 420 {
 421         struct nullb_device *dev = to_nullb_device(item);
 422
 423         null_free_device_storage(dev, false);
 424         null_free_dev(dev);
 425 }
 426
 427 static struct configfs_item_operations nullb_device_ops = {
 428         .release        = nullb_device_release,
 429 };
 430
 431 static const struct config_item_type nullb_device_type = {
 432         .ct_item_ops    = &nullb_device_ops,
 433         .ct_attrs       = nullb_device_attrs,
 434         .ct_owner       = THIS_MODULE,
 435 };
 436
 437 static struct
 438 config_item *nullb_group_make_item(struct config_group *group, const char *name)
 439 {
 440         struct nullb_device *dev;
 441
 442         dev = null_alloc_dev();
 443         if (!dev)
 444                 return ERR_PTR(-ENOMEM);
 445
 446         config_item_init_type_name(&dev->item, name, &nullb_device_type);
 447
 448         return &dev->item;
 449 }
 450
 451 static void
 452 nullb_group_drop_item(struct config_group *group, struct config_item *item)
 453 {
 454         struct nullb_device *dev = to_nullb_device(item);
 455
 456         if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 457                 mutex_lock(&lock);
 458                 dev->power = false;
 459                 null_del_dev(dev->nullb);
 460                 mutex_unlock(&lock);
 461         }
 462
 463         config_item_put(item);
 464 }
 465
 466 static ssize_t memb_group_features_show(struct config_item *item, char *page)
 467 {
 468         return snprintf(page, PAGE_SIZE, "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size\n");
 469 }
 470
 471 CONFIGFS_ATTR_RO(memb_group_, features);
 472
 473 static struct configfs_attribute *nullb_group_attrs[] = {
 474         &memb_group_attr_features,
 475         NULL,
 476 };
 477
 478 static struct configfs_group_operations nullb_group_ops = {
 479         .make_item      = nullb_group_make_item,
 480         .drop_item      = nullb_group_drop_item,
 481 };
 482
 483 static const struct config_item_type nullb_group_type = {
 484         .ct_group_ops   = &nullb_group_ops,
 485         .ct_attrs       = nullb_group_attrs,
 486         .ct_owner       = THIS_MODULE,
 487 };
 488
 489 static struct configfs_subsystem nullb_subsys = {
 490         .su_group = {
 491                 .cg_item = {
 492                         .ci_namebuf = "nullb",
 493                         .ci_type = &nullb_group_type,
 494                 },
 495         },
 496 };
 497
 498 static inline int null_cache_active(struct nullb *nullb)
 499 {
 500         return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
 501 }
 502
 503 static struct nullb_device *null_alloc_dev(void)
 504 {
 505         struct nullb_device *dev;
 506
 507         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 508         if (!dev)
 509                 return NULL;
 510         INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
 511         INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
 512         if (badblocks_init(&dev->badblocks, 0)) {
 513                 kfree(dev);
 514                 return NULL;
 515         }
 516
 517         dev->size = g_gb * 1024;
 518         dev->completion_nsec = g_completion_nsec;
 519         dev->submit_queues = g_submit_queues;
 520         dev->home_node = g_home_node;
 521         dev->queue_mode = g_queue_mode;
 522         dev->blocksize = g_bs;
 523         dev->irqmode = g_irqmode;
 524         dev->hw_queue_depth = g_hw_queue_depth;
 525         dev->blocking = g_blocking;
 526         dev->use_per_node_hctx = g_use_per_node_hctx;
 527         dev->zoned = g_zoned;
 528         dev->zone_size = g_zone_size;
 529         dev->zone_nr_conv = g_zone_nr_conv;
 530         return dev;
 531 }
 532
 533 static void null_free_dev(struct nullb_device *dev)
 534 {
 535         if (!dev)
 536                 return;
 537
 538         null_zone_exit(dev);
 539         badblocks_exit(&dev->badblocks);
 540         kfree(dev);
 541 }
 542
 543 static void put_tag(struct nullb_queue *nq, unsigned int tag)
 544 {
 545         clear_bit_unlock(tag, nq->tag_map);
 546
 547         if (waitqueue_active(&nq->wait))
 548                 wake_up(&nq->wait);
 549 }
 550
 551 static unsigned int get_tag(struct nullb_queue *nq)
 552 {
 553         unsigned int tag;
 554
 555         do {
 556                 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
 557                 if (tag >= nq->queue_depth)
 558                         return -1U;
 559         } while (test_and_set_bit_lock(tag, nq->tag_map));
 560
 561         return tag;
 562 }
 563
 564 static void free_cmd(struct nullb_cmd *cmd)
 565 {
 566         put_tag(cmd->nq, cmd->tag);
 567 }
 568
 569 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
 570
 571 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
 572 {
 573         struct nullb_cmd *cmd;
 574         unsigned int tag;
 575
 576         tag = get_tag(nq);
 577         if (tag != -1U) {
 578                 cmd = &nq->cmds[tag];
 579                 cmd->tag = tag;
 580                 cmd->nq = nq;
 581                 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
 582                         hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
 583                                      HRTIMER_MODE_REL);
 584                         cmd->timer.function = null_cmd_timer_expired;
 585                 }
 586                 return cmd;
 587         }
 588
 589         return NULL;
 590 }
 591
 592 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
 593 {
 594         struct nullb_cmd *cmd;
 595         DEFINE_WAIT(wait);
 596
 597         cmd = __alloc_cmd(nq);
 598         if (cmd || !can_wait)
 599                 return cmd;
 600
 601         do {
 602                 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
 603                 cmd = __alloc_cmd(nq);
 604                 if (cmd)
 605                         break;
 606
 607                 io_schedule();
 608         } while (1);
 609
 610         finish_wait(&nq->wait, &wait);
 611         return cmd;
 612 }
 613
 614 static void end_cmd(struct nullb_cmd *cmd)
 615 {
 616         int queue_mode = cmd->nq->dev->queue_mode;
 617
 618         switch (queue_mode)  {
 619         case NULL_Q_MQ:
 620                 blk_mq_end_request(cmd->rq, cmd->error);
 621                 return;
 622         case NULL_Q_BIO:
 623                 cmd->bio->bi_status = cmd->error;
 624                 bio_endio(cmd->bio);
 625                 break;
 626         }
 627
 628         free_cmd(cmd);
 629 }
 630
 631 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 632 {
 633         end_cmd(container_of(timer, struct nullb_cmd, timer));
 634
 635         return HRTIMER_NORESTART;
 636 }
 637
 638 static void null_cmd_end_timer(struct nullb_cmd *cmd)
 639 {
 640         ktime_t kt = cmd->nq->dev->completion_nsec;
 641
 642         hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 643 }
 644
 645 static void null_complete_rq(struct request *rq)
 646 {
 647         end_cmd(blk_mq_rq_to_pdu(rq));
 648 }
 649
 650 static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
 651 {
 652         struct nullb_page *t_page;
 653
 654         t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
 655         if (!t_page)
 656                 goto out;
 657
 658         t_page->page = alloc_pages(gfp_flags, 0);
 659         if (!t_page->page)
 660                 goto out_freepage;
 661
 662         memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
 663         return t_page;
 664 out_freepage:
 665         kfree(t_page);
 666 out:
 667         return NULL;
 668 }
 669
 670 static void null_free_page(struct nullb_page *t_page)
 671 {
 672         __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
 673         if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
 674                 return;
 675         __free_page(t_page->page);
 676         kfree(t_page);
 677 }
 678
 679 static bool null_page_empty(struct nullb_page *page)
 680 {
 681         int size = MAP_SZ - 2;
 682
 683         return find_first_bit(page->bitmap, size) == size;
 684 }
 685
 686 static void null_free_sector(struct nullb *nullb, sector_t sector,
 687         bool is_cache)
 688 {
 689         unsigned int sector_bit;
 690         u64 idx;
 691         struct nullb_page *t_page, *ret;
 692         struct radix_tree_root *root;
 693
 694         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 695         idx = sector >> PAGE_SECTORS_SHIFT;
 696         sector_bit = (sector & SECTOR_MASK);
 697
 698         t_page = radix_tree_lookup(root, idx);
 699         if (t_page) {
 700                 __clear_bit(sector_bit, t_page->bitmap);
 701
 702                 if (null_page_empty(t_page)) {
 703                         ret = radix_tree_delete_item(root, idx, t_page);
 704                         WARN_ON(ret != t_page);
 705                         null_free_page(ret);
 706                         if (is_cache)
 707                                 nullb->dev->curr_cache -= PAGE_SIZE;
 708                 }
 709         }
 710 }
 711
 712 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
 713         struct nullb_page *t_page, bool is_cache)
 714 {
 715         struct radix_tree_root *root;
 716
 717         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 718
 719         if (radix_tree_insert(root, idx, t_page)) {
 720                 null_free_page(t_page);
 721                 t_page = radix_tree_lookup(root, idx);
 722                 WARN_ON(!t_page || t_page->page->index != idx);
 723         } else if (is_cache)
 724                 nullb->dev->curr_cache += PAGE_SIZE;
 725
 726         return t_page;
 727 }
 728
 729 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
 730 {
 731         unsigned long pos = 0;
 732         int nr_pages;
 733         struct nullb_page *ret, *t_pages[FREE_BATCH];
 734         struct radix_tree_root *root;
 735
 736         root = is_cache ? &dev->cache : &dev->data;
 737
 738         do {
 739                 int i;
 740
 741                 nr_pages = radix_tree_gang_lookup(root,
 742                                 (void **)t_pages, pos, FREE_BATCH);
 743
 744                 for (i = 0; i < nr_pages; i++) {
 745                         pos = t_pages[i]->page->index;
 746                         ret = radix_tree_delete_item(root, pos, t_pages[i]);
 747                         WARN_ON(ret != t_pages[i]);
 748                         null_free_page(ret);
 749                 }
 750
 751                 pos++;
 752         } while (nr_pages == FREE_BATCH);
 753
 754         if (is_cache)
 755                 dev->curr_cache = 0;
 756 }
 757
 758 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
 759         sector_t sector, bool for_write, bool is_cache)
 760 {
 761         unsigned int sector_bit;
 762         u64 idx;
 763         struct nullb_page *t_page;
 764         struct radix_tree_root *root;
 765
 766         idx = sector >> PAGE_SECTORS_SHIFT;
 767         sector_bit = (sector & SECTOR_MASK);
 768
 769         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 770         t_page = radix_tree_lookup(root, idx);
 771         WARN_ON(t_page && t_page->page->index != idx);
 772
 773         if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
 774                 return t_page;
 775
 776         return NULL;
 777 }
 778
 779 static struct nullb_page *null_lookup_page(struct nullb *nullb,
 780         sector_t sector, bool for_write, bool ignore_cache)
 781 {
 782         struct nullb_page *page = NULL;
 783
 784         if (!ignore_cache)
 785                 page = __null_lookup_page(nullb, sector, for_write, true);
 786         if (page)
 787                 return page;
 788         return __null_lookup_page(nullb, sector, for_write, false);
 789 }
 790
 791 static struct nullb_page *null_insert_page(struct nullb *nullb,
 792                                            sector_t sector, bool ignore_cache)
 793         __releases(&nullb->lock)
 794         __acquires(&nullb->lock)
 795 {
 796         u64 idx;
 797         struct nullb_page *t_page;
 798
 799         t_page = null_lookup_page(nullb, sector, true, ignore_cache);
 800         if (t_page)
 801                 return t_page;
 802
 803         spin_unlock_irq(&nullb->lock);
 804
 805         t_page = null_alloc_page(GFP_NOIO);
 806         if (!t_page)
 807                 goto out_lock;
 808
 809         if (radix_tree_preload(GFP_NOIO))
 810                 goto out_freepage;
 811
 812         spin_lock_irq(&nullb->lock);
 813         idx = sector >> PAGE_SECTORS_SHIFT;
 814         t_page->page->index = idx;
 815         t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
 816         radix_tree_preload_end();
 817
 818         return t_page;
 819 out_freepage:
 820         null_free_page(t_page);
 821 out_lock:
 822         spin_lock_irq(&nullb->lock);
 823         return null_lookup_page(nullb, sector, true, ignore_cache);
 824 }
 825
 826 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
 827 {
 828         int i;
 829         unsigned int offset;
 830         u64 idx;
 831         struct nullb_page *t_page, *ret;
 832         void *dst, *src;
 833
 834         idx = c_page->page->index;
 835
 836         t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
 837
 838         __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
 839         if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
 840                 null_free_page(c_page);
 841                 if (t_page && null_page_empty(t_page)) {
 842                         ret = radix_tree_delete_item(&nullb->dev->data,
 843                                 idx, t_page);
 844                         null_free_page(t_page);
 845                 }
 846                 return 0;
 847         }
 848
 849         if (!t_page)
 850                 return -ENOMEM;
 851
 852         src = kmap_atomic(c_page->page);
 853         dst = kmap_atomic(t_page->page);
 854
 855         for (i = 0; i < PAGE_SECTORS;
 856                         i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
 857                 if (test_bit(i, c_page->bitmap)) {
 858                         offset = (i << SECTOR_SHIFT);
 859                         memcpy(dst + offset, src + offset,
 860                                 nullb->dev->blocksize);
 861                         __set_bit(i, t_page->bitmap);
 862                 }
 863         }
 864
 865         kunmap_atomic(dst);
 866         kunmap_atomic(src);
 867
 868         ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
 869         null_free_page(ret);
 870         nullb->dev->curr_cache -= PAGE_SIZE;
 871
 872         return 0;
 873 }
 874
 875 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
 876 {
 877         int i, err, nr_pages;
 878         struct nullb_page *c_pages[FREE_BATCH];
 879         unsigned long flushed = 0, one_round;
 880
 881 again:
 882         if ((nullb->dev->cache_size * 1024 * 1024) >
 883              nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
 884                 return 0;
 885
 886         nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
 887                         (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
 888         /*
 889          * nullb_flush_cache_page could unlock before using the c_pages. To
 890          * avoid race, we don't allow page free
 891          */
 892         for (i = 0; i < nr_pages; i++) {
 893                 nullb->cache_flush_pos = c_pages[i]->page->index;
 894                 /*
 895                  * We found the page which is being flushed to disk by other
 896                  * threads
 897                  */
 898                 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
 899                         c_pages[i] = NULL;
 900                 else
 901                         __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
 902         }
 903
 904         one_round = 0;
 905         for (i = 0; i < nr_pages; i++) {
 906                 if (c_pages[i] == NULL)
 907                         continue;
 908                 err = null_flush_cache_page(nullb, c_pages[i]);
 909                 if (err)
 910                         return err;
 911                 one_round++;
 912         }
 913         flushed += one_round << PAGE_SHIFT;
 914
 915         if (n > flushed) {
 916                 if (nr_pages == 0)
 917                         nullb->cache_flush_pos = 0;
 918                 if (one_round == 0) {
 919                         /* give other threads a chance */
 920                         spin_unlock_irq(&nullb->lock);
 921                         spin_lock_irq(&nullb->lock);
 922                 }
 923                 goto again;
 924         }
 925         return 0;
 926 }
 927
 928 static int copy_to_nullb(struct nullb *nullb, struct page *source,
 929         unsigned int off, sector_t sector, size_t n, bool is_fua)
 930 {
 931         size_t temp, count = 0;
 932         unsigned int offset;
 933         struct nullb_page *t_page;
 934         void *dst, *src;
 935
 936         while (count < n) {
 937                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
 938
 939                 if (null_cache_active(nullb) && !is_fua)
 940                         null_make_cache_space(nullb, PAGE_SIZE);
 941
 942                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
 943                 t_page = null_insert_page(nullb, sector,
 944                         !null_cache_active(nullb) || is_fua);
 945                 if (!t_page)
 946                         return -ENOSPC;
 947
 948                 src = kmap_atomic(source);
 949                 dst = kmap_atomic(t_page->page);
 950                 memcpy(dst + offset, src + off + count, temp);
 951                 kunmap_atomic(dst);
 952                 kunmap_atomic(src);
 953
 954                 __set_bit(sector & SECTOR_MASK, t_page->bitmap);
 955
 956                 if (is_fua)
 957                         null_free_sector(nullb, sector, true);
 958
 959                 count += temp;
 960                 sector += temp >> SECTOR_SHIFT;
 961         }
 962         return 0;
 963 }
 964
 965 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
 966         unsigned int off, sector_t sector, size_t n)
 967 {
 968         size_t temp, count = 0;
 969         unsigned int offset;
 970         struct nullb_page *t_page;
 971         void *dst, *src;
 972
 973         while (count < n) {
 974                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
 975
 976                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
 977                 t_page = null_lookup_page(nullb, sector, false,
 978                         !null_cache_active(nullb));
 979
 980                 dst = kmap_atomic(dest);
 981                 if (!t_page) {
 982                         memset(dst + off + count, 0, temp);
 983                         goto next;
 984                 }
 985                 src = kmap_atomic(t_page->page);
 986                 memcpy(dst + off + count, src + offset, temp);
 987                 kunmap_atomic(src);
 988 next:
 989                 kunmap_atomic(dst);
 990
 991                 count += temp;
 992                 sector += temp >> SECTOR_SHIFT;
 993         }
 994         return 0;
 995 }
 996
 997 static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)
 998 {
 999         size_t temp;
1000
1001         spin_lock_irq(&nullb->lock);
1002         while (n > 0) {
1003                 temp = min_t(size_t, n, nullb->dev->blocksize);
1004                 null_free_sector(nullb, sector, false);
1005                 if (null_cache_active(nullb))
1006                         null_free_sector(nullb, sector, true);
1007                 sector += temp >> SECTOR_SHIFT;
1008                 n -= temp;
1009         }
1010         spin_unlock_irq(&nullb->lock);
1011 }
1012
1013 static int null_handle_flush(struct nullb *nullb)
1014 {
1015         int err;
1016
1017         if (!null_cache_active(nullb))
1018                 return 0;
1019
1020         spin_lock_irq(&nullb->lock);
1021         while (true) {
1022                 err = null_make_cache_space(nullb,
1023                         nullb->dev->cache_size * 1024 * 1024);
1024                 if (err || nullb->dev->curr_cache == 0)
1025                         break;
1026         }
1027
1028         WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1029         spin_unlock_irq(&nullb->lock);
1030         return err;
1031 }
1032
1033 static int null_transfer(struct nullb *nullb, struct page *page,
1034         unsigned int len, unsigned int off, bool is_write, sector_t sector,
1035         bool is_fua)
1036 {
1037         int err = 0;
1038
1039         if (!is_write) {
1040                 err = copy_from_nullb(nullb, page, off, sector, len);
1041                 flush_dcache_page(page);
1042         } else {
1043                 flush_dcache_page(page);
1044                 err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1045         }
1046
1047         return err;
1048 }
1049
1050 static int null_handle_rq(struct nullb_cmd *cmd)
1051 {
1052         struct request *rq = cmd->rq;
1053         struct nullb *nullb = cmd->nq->dev->nullb;
1054         int err;
1055         unsigned int len;
1056         sector_t sector;
1057         struct req_iterator iter;
1058         struct bio_vec bvec;
1059
1060         sector = blk_rq_pos(rq);
1061
1062         if (req_op(rq) == REQ_OP_DISCARD) {
1063                 null_handle_discard(nullb, sector, blk_rq_bytes(rq));
1064                 return 0;
1065         }
1066
1067         spin_lock_irq(&nullb->lock);
1068         rq_for_each_segment(bvec, rq, iter) {
1069                 len = bvec.bv_len;
1070                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1071                                      op_is_write(req_op(rq)), sector,
1072                                      req_op(rq) & REQ_FUA);
1073                 if (err) {
1074                         spin_unlock_irq(&nullb->lock);
1075                         return err;
1076                 }
1077                 sector += len >> SECTOR_SHIFT;
1078         }
1079         spin_unlock_irq(&nullb->lock);
1080
1081         return 0;
1082 }
1083
1084 static int null_handle_bio(struct nullb_cmd *cmd)
1085 {
1086         struct bio *bio = cmd->bio;
1087         struct nullb *nullb = cmd->nq->dev->nullb;
1088         int err;
1089         unsigned int len;
1090         sector_t sector;
1091         struct bio_vec bvec;
1092         struct bvec_iter iter;
1093
1094         sector = bio->bi_iter.bi_sector;
1095
1096         if (bio_op(bio) == REQ_OP_DISCARD) {
1097                 null_handle_discard(nullb, sector,
1098                         bio_sectors(bio) << SECTOR_SHIFT);
1099                 return 0;
1100         }
1101
1102         spin_lock_irq(&nullb->lock);
1103         bio_for_each_segment(bvec, bio, iter) {
1104                 len = bvec.bv_len;
1105                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1106                                      op_is_write(bio_op(bio)), sector,
1107                                      bio->bi_opf & REQ_FUA);
1108                 if (err) {
1109                         spin_unlock_irq(&nullb->lock);
1110                         return err;
1111                 }
1112                 sector += len >> SECTOR_SHIFT;
1113         }
1114         spin_unlock_irq(&nullb->lock);
1115         return 0;
1116 }
1117
1118 static void null_stop_queue(struct nullb *nullb)
1119 {
1120         struct request_queue *q = nullb->q;
1121
1122         if (nullb->dev->queue_mode == NULL_Q_MQ)
1123                 blk_mq_stop_hw_queues(q);
1124 }
1125
1126 static void null_restart_queue_async(struct nullb *nullb)
1127 {
1128         struct request_queue *q = nullb->q;
1129
1130         if (nullb->dev->queue_mode == NULL_Q_MQ)
1131                 blk_mq_start_stopped_hw_queues(q, true);
1132 }
1133
1134 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
1135 {
1136         struct nullb_device *dev = cmd->nq->dev;
1137         struct nullb *nullb = dev->nullb;
1138         int err = 0;
1139
1140         if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
1141                 struct request *rq = cmd->rq;
1142
1143                 if (!hrtimer_active(&nullb->bw_timer))
1144                         hrtimer_restart(&nullb->bw_timer);
1145
1146                 if (atomic_long_sub_return(blk_rq_bytes(rq),
1147                                 &nullb->cur_bytes) < 0) {
1148                         null_stop_queue(nullb);
1149                         /* race with timer */
1150                         if (atomic_long_read(&nullb->cur_bytes) > 0)
1151                                 null_restart_queue_async(nullb);
1152                         /* requeue request */
1153                         return BLK_STS_DEV_RESOURCE;
1154                 }
1155         }
1156
1157         if (nullb->dev->badblocks.shift != -1) {
1158                 int bad_sectors;
1159                 sector_t sector, size, first_bad;
1160                 bool is_flush = true;
1161
1162                 if (dev->queue_mode == NULL_Q_BIO &&
1163                                 bio_op(cmd->bio) != REQ_OP_FLUSH) {
1164                         is_flush = false;
1165                         sector = cmd->bio->bi_iter.bi_sector;
1166                         size = bio_sectors(cmd->bio);
1167                 }
1168                 if (dev->queue_mode != NULL_Q_BIO &&
1169                                 req_op(cmd->rq) != REQ_OP_FLUSH) {
1170                         is_flush = false;
1171                         sector = blk_rq_pos(cmd->rq);
1172                         size = blk_rq_sectors(cmd->rq);
1173                 }
1174                 if (!is_flush && badblocks_check(&nullb->dev->badblocks, sector,
1175                                 size, &first_bad, &bad_sectors)) {
1176                         cmd->error = BLK_STS_IOERR;
1177                         goto out;
1178                 }
1179         }
1180
1181         if (dev->memory_backed) {
1182                 if (dev->queue_mode == NULL_Q_BIO) {
1183                         if (bio_op(cmd->bio) == REQ_OP_FLUSH)
1184                                 err = null_handle_flush(nullb);
1185                         else
1186                                 err = null_handle_bio(cmd);
1187                 } else {
1188                         if (req_op(cmd->rq) == REQ_OP_FLUSH)
1189                                 err = null_handle_flush(nullb);
1190                         else
1191                                 err = null_handle_rq(cmd);
1192                 }
1193         }
1194         cmd->error = errno_to_blk_status(err);
1195
1196         if (!cmd->error && dev->zoned) {
1197                 sector_t sector;
1198                 unsigned int nr_sectors;
1199                 int op;
1200
1201                 if (dev->queue_mode == NULL_Q_BIO) {
1202                         op = bio_op(cmd->bio);
1203                         sector = cmd->bio->bi_iter.bi_sector;
1204                         nr_sectors = cmd->bio->bi_iter.bi_size >> 9;
1205                 } else {
1206                         op = req_op(cmd->rq);
1207                         sector = blk_rq_pos(cmd->rq);
1208                         nr_sectors = blk_rq_sectors(cmd->rq);
1209                 }
1210
1211                 if (op == REQ_OP_WRITE)
1212                         null_zone_write(cmd, sector, nr_sectors);
1213                 else if (op == REQ_OP_ZONE_RESET)
1214                         null_zone_reset(cmd, sector);
1215         }
1216 out:
1217         /* Complete IO by inline, softirq or timer */
1218         switch (dev->irqmode) {
1219         case NULL_IRQ_SOFTIRQ:
1220                 switch (dev->queue_mode)  {
1221                 case NULL_Q_MQ:
1222                         blk_mq_complete_request(cmd->rq);
1223                         break;
1224                 case NULL_Q_BIO:
1225                         /*
1226                          * XXX: no proper submitting cpu information available.
1227                          */
1228                         end_cmd(cmd);
1229                         break;
1230                 }
1231                 break;
1232         case NULL_IRQ_NONE:
1233                 end_cmd(cmd);
1234                 break;
1235         case NULL_IRQ_TIMER:
1236                 null_cmd_end_timer(cmd);
1237                 break;
1238         }
1239         return BLK_STS_OK;
1240 }
1241
1242 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1243 {
1244         struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1245         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1246         unsigned int mbps = nullb->dev->mbps;
1247
1248         if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1249                 return HRTIMER_NORESTART;
1250
1251         atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1252         null_restart_queue_async(nullb);
1253
1254         hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1255
1256         return HRTIMER_RESTART;
1257 }
1258
1259 static void nullb_setup_bwtimer(struct nullb *nullb)
1260 {
1261         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1262
1263         hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1264         nullb->bw_timer.function = nullb_bwtimer_fn;
1265         atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1266         hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1267 }
1268
1269 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
1270 {
1271         int index = 0;
1272
1273         if (nullb->nr_queues != 1)
1274                 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
1275
1276         return &nullb->queues[index];
1277 }
1278
1279 static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
1280 {
1281         struct nullb *nullb = q->queuedata;
1282         struct nullb_queue *nq = nullb_to_queue(nullb);
1283         struct nullb_cmd *cmd;
1284
1285         cmd = alloc_cmd(nq, 1);
1286         cmd->bio = bio;
1287
1288         null_handle_cmd(cmd);
1289         return BLK_QC_T_NONE;
1290 }
1291
1292 static bool should_timeout_request(struct request *rq)
1293 {
1294 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1295         if (g_timeout_str[0])
1296                 return should_fail(&null_timeout_attr, 1);
1297 #endif
1298         return false;
1299 }
1300
1301 static bool should_requeue_request(struct request *rq)
1302 {
1303 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1304         if (g_requeue_str[0])
1305                 return should_fail(&null_requeue_attr, 1);
1306 #endif
1307         return false;
1308 }
1309
1310 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
1311 {
1312         pr_info("null: rq %p timed out\n", rq);
1313         blk_mq_complete_request(rq);
1314         return BLK_EH_DONE;
1315 }
1316
1317 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1318                          const struct blk_mq_queue_data *bd)
1319 {
1320         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1321         struct nullb_queue *nq = hctx->driver_data;
1322
1323         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1324
1325         if (nq->dev->irqmode == NULL_IRQ_TIMER) {
1326                 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1327                 cmd->timer.function = null_cmd_timer_expired;
1328         }
1329         cmd->rq = bd->rq;
1330         cmd->nq = nq;
1331
1332         blk_mq_start_request(bd->rq);
1333
1334         if (should_requeue_request(bd->rq)) {
1335                 /*
1336                  * Alternate between hitting the core BUSY path, and the
1337                  * driver driven requeue path
1338                  */
1339                 nq->requeue_selection++;
1340                 if (nq->requeue_selection & 1)
1341                         return BLK_STS_RESOURCE;
1342                 else {
1343                         blk_mq_requeue_request(bd->rq, true);
1344                         return BLK_STS_OK;
1345                 }
1346         }
1347         if (should_timeout_request(bd->rq))
1348                 return BLK_STS_OK;
1349
1350         return null_handle_cmd(cmd);
1351 }
1352
1353 static const struct blk_mq_ops null_mq_ops = {
1354         .queue_rq       = null_queue_rq,
1355         .complete       = null_complete_rq,
1356         .timeout        = null_timeout_rq,
1357 };
1358
1359 static void cleanup_queue(struct nullb_queue *nq)
1360 {
1361         kfree(nq->tag_map);
1362         kfree(nq->cmds);
1363 }
1364
1365 static void cleanup_queues(struct nullb *nullb)
1366 {
1367         int i;
1368
1369         for (i = 0; i < nullb->nr_queues; i++)
1370                 cleanup_queue(&nullb->queues[i]);
1371
1372         kfree(nullb->queues);
1373 }
1374
1375 static void null_del_dev(struct nullb *nullb)
1376 {
1377         struct nullb_device *dev = nullb->dev;
1378
1379         ida_simple_remove(&nullb_indexes, nullb->index);
1380
1381         list_del_init(&nullb->list);
1382
1383         del_gendisk(nullb->disk);
1384
1385         if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1386                 hrtimer_cancel(&nullb->bw_timer);
1387                 atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1388                 null_restart_queue_async(nullb);
1389         }
1390
1391         blk_cleanup_queue(nullb->q);
1392         if (dev->queue_mode == NULL_Q_MQ &&
1393             nullb->tag_set == &nullb->__tag_set)
1394                 blk_mq_free_tag_set(nullb->tag_set);
1395         put_disk(nullb->disk);
1396         cleanup_queues(nullb);
1397         if (null_cache_active(nullb))
1398                 null_free_device_storage(nullb->dev, true);
1399         kfree(nullb);
1400         dev->nullb = NULL;
1401 }
1402
1403 static void null_config_discard(struct nullb *nullb)
1404 {
1405         if (nullb->dev->discard == false)
1406                 return;
1407         nullb->q->limits.discard_granularity = nullb->dev->blocksize;
1408         nullb->q->limits.discard_alignment = nullb->dev->blocksize;
1409         blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
1410         blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q);
1411 }
1412
1413 static int null_open(struct block_device *bdev, fmode_t mode)
1414 {
1415         return 0;
1416 }
1417
1418 static void null_release(struct gendisk *disk, fmode_t mode)
1419 {
1420 }
1421
1422 static const struct block_device_operations null_fops = {
1423         .owner =        THIS_MODULE,
1424         .open =         null_open,
1425         .release =      null_release,
1426         .report_zones = null_zone_report,
1427 };
1428
1429 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1430 {
1431         BUG_ON(!nullb);
1432         BUG_ON(!nq);
1433
1434         init_waitqueue_head(&nq->wait);
1435         nq->queue_depth = nullb->queue_depth;
1436         nq->dev = nullb->dev;
1437 }
1438
1439 static void null_init_queues(struct nullb *nullb)
1440 {
1441         struct request_queue *q = nullb->q;
1442         struct blk_mq_hw_ctx *hctx;
1443         struct nullb_queue *nq;
1444         int i;
1445
1446         queue_for_each_hw_ctx(q, hctx, i) {
1447                 if (!hctx->nr_ctx || !hctx->tags)
1448                         continue;
1449                 nq = &nullb->queues[i];
1450                 hctx->driver_data = nq;
1451                 null_init_queue(nullb, nq);
1452                 nullb->nr_queues++;
1453         }
1454 }
1455
1456 static int setup_commands(struct nullb_queue *nq)
1457 {
1458         struct nullb_cmd *cmd;
1459         int i, tag_size;
1460
1461         nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);
1462         if (!nq->cmds)
1463                 return -ENOMEM;
1464
1465         tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
1466         nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL);
1467         if (!nq->tag_map) {
1468                 kfree(nq->cmds);
1469                 return -ENOMEM;
1470         }
1471
1472         for (i = 0; i < nq->queue_depth; i++) {
1473                 cmd = &nq->cmds[i];
1474                 INIT_LIST_HEAD(&cmd->list);
1475                 cmd->ll_list.next = NULL;
1476                 cmd->tag = -1U;
1477         }
1478
1479         return 0;
1480 }
1481
1482 static int setup_queues(struct nullb *nullb)
1483 {
1484         nullb->queues = kcalloc(nullb->dev->submit_queues,
1485                                 sizeof(struct nullb_queue),
1486                                 GFP_KERNEL);
1487         if (!nullb->queues)
1488                 return -ENOMEM;
1489
1490         nullb->nr_queues = 0;
1491         nullb->queue_depth = nullb->dev->hw_queue_depth;
1492
1493         return 0;
1494 }
1495
1496 static int init_driver_queues(struct nullb *nullb)
1497 {
1498         struct nullb_queue *nq;
1499         int i, ret = 0;
1500
1501         for (i = 0; i < nullb->dev->submit_queues; i++) {
1502                 nq = &nullb->queues[i];
1503
1504                 null_init_queue(nullb, nq);
1505
1506                 ret = setup_commands(nq);
1507                 if (ret)
1508                         return ret;
1509                 nullb->nr_queues++;
1510         }
1511         return 0;
1512 }
1513
1514 static int null_gendisk_register(struct nullb *nullb)
1515 {
1516         struct gendisk *disk;
1517         sector_t size;
1518
1519         disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
1520         if (!disk)
1521                 return -ENOMEM;
1522         size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
1523         set_capacity(disk, size >> 9);
1524
1525         disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
1526         disk->major             = null_major;
1527         disk->first_minor       = nullb->index;
1528         disk->fops              = &null_fops;
1529         disk->private_data      = nullb;
1530         disk->queue             = nullb->q;
1531         strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1532
1533         if (nullb->dev->zoned) {
1534                 int ret = blk_revalidate_disk_zones(disk);
1535
1536                 if (ret != 0)
1537                         return ret;
1538         }
1539
1540         add_disk(disk);
1541         return 0;
1542 }
1543
1544 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
1545 {
1546         set->ops = &null_mq_ops;
1547         set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
1548                                                 g_submit_queues;
1549         set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
1550                                                 g_hw_queue_depth;
1551         set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
1552         set->cmd_size   = sizeof(struct nullb_cmd);
1553         set->flags = BLK_MQ_F_SHOULD_MERGE;
1554         if (g_no_sched)
1555                 set->flags |= BLK_MQ_F_NO_SCHED;
1556         set->driver_data = NULL;
1557
1558         if ((nullb && nullb->dev->blocking) || g_blocking)
1559                 set->flags |= BLK_MQ_F_BLOCKING;
1560
1561         return blk_mq_alloc_tag_set(set);
1562 }
1563
1564 static void null_validate_conf(struct nullb_device *dev)
1565 {
1566         dev->blocksize = round_down(dev->blocksize, 512);
1567         dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1568
1569         if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
1570                 if (dev->submit_queues != nr_online_nodes)
1571                         dev->submit_queues = nr_online_nodes;
1572         } else if (dev->submit_queues > nr_cpu_ids)
1573                 dev->submit_queues = nr_cpu_ids;
1574         else if (dev->submit_queues == 0)
1575                 dev->submit_queues = 1;
1576
1577         dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
1578         dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1579
1580         /* Do memory allocation, so set blocking */
1581         if (dev->memory_backed)
1582                 dev->blocking = true;
1583         else /* cache is meaningless */
1584                 dev->cache_size = 0;
1585         dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1586                                                 dev->cache_size);
1587         dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1588         /* can not stop a queue */
1589         if (dev->queue_mode == NULL_Q_BIO)
1590                 dev->mbps = 0;
1591 }
1592
1593 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1594 static bool __null_setup_fault(struct fault_attr *attr, char *str)
1595 {
1596         if (!str[0])
1597                 return true;
1598
1599         if (!setup_fault_attr(attr, str))
1600                 return false;
1601
1602         attr->verbose = 0;
1603         return true;
1604 }
1605 #endif
1606
1607 static bool null_setup_fault(void)
1608 {
1609 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1610         if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
1611                 return false;
1612         if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
1613                 return false;
1614 #endif
1615         return true;
1616 }
1617
1618 static int null_add_dev(struct nullb_device *dev)
1619 {
1620         struct nullb *nullb;
1621         int rv;
1622
1623         null_validate_conf(dev);
1624
1625         nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1626         if (!nullb) {
1627                 rv = -ENOMEM;
1628                 goto out;
1629         }
1630         nullb->dev = dev;
1631         dev->nullb = nullb;
1632
1633         spin_lock_init(&nullb->lock);
1634
1635         rv = setup_queues(nullb);
1636         if (rv)
1637                 goto out_free_nullb;
1638
1639         if (dev->queue_mode == NULL_Q_MQ) {
1640                 if (shared_tags) {
1641                         nullb->tag_set = &tag_set;
1642                         rv = 0;
1643                 } else {
1644                         nullb->tag_set = &nullb->__tag_set;
1645                         rv = null_init_tag_set(nullb, nullb->tag_set);
1646                 }
1647
1648                 if (rv)
1649                         goto out_cleanup_queues;
1650
1651                 if (!null_setup_fault())
1652                         goto out_cleanup_queues;
1653
1654                 nullb->tag_set->timeout = 5 * HZ;
1655                 nullb->q = blk_mq_init_queue(nullb->tag_set);
1656                 if (IS_ERR(nullb->q)) {
1657                         rv = -ENOMEM;
1658                         goto out_cleanup_tags;
1659                 }
1660                 null_init_queues(nullb);
1661         } else if (dev->queue_mode == NULL_Q_BIO) {
1662                 nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node);
1663                 if (!nullb->q) {
1664                         rv = -ENOMEM;
1665                         goto out_cleanup_queues;
1666                 }
1667                 blk_queue_make_request(nullb->q, null_queue_bio);
1668                 rv = init_driver_queues(nullb);
1669                 if (rv)
1670                         goto out_cleanup_blk_queue;
1671         }
1672
1673         if (dev->mbps) {
1674                 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
1675                 nullb_setup_bwtimer(nullb);
1676         }
1677
1678         if (dev->cache_size > 0) {
1679                 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
1680                 blk_queue_write_cache(nullb->q, true, true);
1681         }
1682
1683         if (dev->zoned) {
1684                 rv = null_zone_init(dev);
1685                 if (rv)
1686                         goto out_cleanup_blk_queue;
1687
1688                 blk_queue_chunk_sectors(nullb->q, dev->zone_size_sects);
1689                 nullb->q->limits.zoned = BLK_ZONED_HM;
1690         }
1691
1692         nullb->q->queuedata = nullb;
1693         blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
1694         blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);
1695
1696         mutex_lock(&lock);
1697         nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
1698         dev->index = nullb->index;
1699         mutex_unlock(&lock);
1700
1701         blk_queue_logical_block_size(nullb->q, dev->blocksize);
1702         blk_queue_physical_block_size(nullb->q, dev->blocksize);
1703
1704         null_config_discard(nullb);
1705
1706         sprintf(nullb->disk_name, "nullb%d", nullb->index);
1707
1708         rv = null_gendisk_register(nullb);
1709         if (rv)
1710                 goto out_cleanup_zone;
1711
1712         mutex_lock(&lock);
1713         list_add_tail(&nullb->list, &nullb_list);
1714         mutex_unlock(&lock);
1715
1716         return 0;
1717 out_cleanup_zone:
1718         if (dev->zoned)
1719                 null_zone_exit(dev);
1720 out_cleanup_blk_queue:
1721         blk_cleanup_queue(nullb->q);
1722 out_cleanup_tags:
1723         if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
1724                 blk_mq_free_tag_set(nullb->tag_set);
1725 out_cleanup_queues:
1726         cleanup_queues(nullb);
1727 out_free_nullb:
1728         kfree(nullb);
1729 out:
1730         return rv;
1731 }
1732
1733 static int __init null_init(void)
1734 {
1735         int ret = 0;
1736         unsigned int i;
1737         struct nullb *nullb;
1738         struct nullb_device *dev;
1739
1740         if (g_bs > PAGE_SIZE) {
1741                 pr_warn("null_blk: invalid block size\n");
1742                 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
1743                 g_bs = PAGE_SIZE;
1744         }
1745
1746         if (!is_power_of_2(g_zone_size)) {
1747                 pr_err("null_blk: zone_size must be power-of-two\n");
1748                 return -EINVAL;
1749         }
1750
1751         if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
1752                 pr_err("null_blk: invalid home_node value\n");
1753                 g_home_node = NUMA_NO_NODE;
1754         }
1755
1756         if (g_queue_mode == NULL_Q_RQ) {
1757                 pr_err("null_blk: legacy IO path no longer available\n");
1758                 return -EINVAL;
1759         }
1760         if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
1761                 if (g_submit_queues != nr_online_nodes) {
1762                         pr_warn("null_blk: submit_queues param is set to %u.\n",
1763                                                         nr_online_nodes);
1764                         g_submit_queues = nr_online_nodes;
1765                 }
1766         } else if (g_submit_queues > nr_cpu_ids)
1767                 g_submit_queues = nr_cpu_ids;
1768         else if (g_submit_queues <= 0)
1769                 g_submit_queues = 1;
1770
1771         if (g_queue_mode == NULL_Q_MQ && shared_tags) {
1772                 ret = null_init_tag_set(NULL, &tag_set);
1773                 if (ret)
1774                         return ret;
1775         }
1776
1777         config_group_init(&nullb_subsys.su_group);
1778         mutex_init(&nullb_subsys.su_mutex);
1779
1780         ret = configfs_register_subsystem(&nullb_subsys);
1781         if (ret)
1782                 goto err_tagset;
1783
1784         mutex_init(&lock);
1785
1786         null_major = register_blkdev(0, "nullb");
1787         if (null_major < 0) {
1788                 ret = null_major;
1789                 goto err_conf;
1790         }
1791
1792         for (i = 0; i < nr_devices; i++) {
1793                 dev = null_alloc_dev();
1794                 if (!dev) {
1795                         ret = -ENOMEM;
1796                         goto err_dev;
1797                 }
1798                 ret = null_add_dev(dev);
1799                 if (ret) {
1800                         null_free_dev(dev);
1801                         goto err_dev;
1802                 }
1803         }
1804
1805         pr_info("null: module loaded\n");
1806         return 0;
1807
1808 err_dev:
1809         while (!list_empty(&nullb_list)) {
1810                 nullb = list_entry(nullb_list.next, struct nullb, list);
1811                 dev = nullb->dev;
1812                 null_del_dev(nullb);
1813                 null_free_dev(dev);
1814         }
1815         unregister_blkdev(null_major, "nullb");
1816 err_conf:
1817         configfs_unregister_subsystem(&nullb_subsys);
1818 err_tagset:
1819         if (g_queue_mode == NULL_Q_MQ && shared_tags)
1820                 blk_mq_free_tag_set(&tag_set);
1821         return ret;
1822 }
1823
1824 static void __exit null_exit(void)
1825 {
1826         struct nullb *nullb;
1827
1828         configfs_unregister_subsystem(&nullb_subsys);
1829
1830         unregister_blkdev(null_major, "nullb");
1831
1832         mutex_lock(&lock);
1833         while (!list_empty(&nullb_list)) {
1834                 struct nullb_device *dev;
1835
1836                 nullb = list_entry(nullb_list.next, struct nullb, list);
1837                 dev = nullb->dev;
1838                 null_del_dev(nullb);
1839                 null_free_dev(dev);
1840         }
1841         mutex_unlock(&lock);
1842
1843         if (g_queue_mode == NULL_Q_MQ && shared_tags)
1844                 blk_mq_free_tag_set(&tag_set);
1845 }
1846
1847 module_init(null_init);
1848 module_exit(null_exit);
1849
1850 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
1851 MODULE_LICENSE("GPL");