drivers/nvme/host/multipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2017-2018 Christoph Hellwig.
   4  */
   5
   6 #include <linux/backing-dev.h>
   7 #include <linux/moduleparam.h>
   8 #include <linux/vmalloc.h>
   9 #include <trace/events/block.h>
  10 #include "nvme.h"
  11
  12 bool multipath = true;
  13 module_param(multipath, bool, 0444);
  14 MODULE_PARM_DESC(multipath,
  15         "turn on native support for multiple controllers per subsystem");
  16
  17 static const char *nvme_iopolicy_names[] = {
  18         [NVME_IOPOLICY_NUMA]    = "numa",
  19         [NVME_IOPOLICY_RR]      = "round-robin",
  20 };
  21
  22 static int iopolicy = NVME_IOPOLICY_NUMA;
  23
  24 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
  25 {
  26         if (!val)
  27                 return -EINVAL;
  28         if (!strncmp(val, "numa", 4))
  29                 iopolicy = NVME_IOPOLICY_NUMA;
  30         else if (!strncmp(val, "round-robin", 11))
  31                 iopolicy = NVME_IOPOLICY_RR;
  32         else
  33                 return -EINVAL;
  34
  35         return 0;
  36 }
  37
  38 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
  39 {
  40         return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
  41 }
  42
  43 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
  44         &iopolicy, 0644);
  45 MODULE_PARM_DESC(iopolicy,
  46         "Default multipath I/O policy; 'numa' (default) or 'round-robin'");
  47
  48 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
  49 {
  50         subsys->iopolicy = iopolicy;
  51 }
  52
  53 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
  54 {
  55         struct nvme_ns_head *h;
  56
  57         lockdep_assert_held(&subsys->lock);
  58         list_for_each_entry(h, &subsys->nsheads, entry)
  59                 if (h->disk)
  60                         blk_mq_unfreeze_queue(h->disk->queue);
  61 }
  62
  63 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
  64 {
  65         struct nvme_ns_head *h;
  66
  67         lockdep_assert_held(&subsys->lock);
  68         list_for_each_entry(h, &subsys->nsheads, entry)
  69                 if (h->disk)
  70                         blk_mq_freeze_queue_wait(h->disk->queue);
  71 }
  72
  73 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
  74 {
  75         struct nvme_ns_head *h;
  76
  77         lockdep_assert_held(&subsys->lock);
  78         list_for_each_entry(h, &subsys->nsheads, entry)
  79                 if (h->disk)
  80                         blk_freeze_queue_start(h->disk->queue);
  81 }
  82
  83 void nvme_failover_req(struct request *req)
  84 {
  85         struct nvme_ns *ns = req->q->queuedata;
  86         u16 status = nvme_req(req)->status & 0x7ff;
  87         unsigned long flags;
  88         struct bio *bio;
  89
  90         nvme_mpath_clear_current_path(ns);
  91
  92         /*
  93          * If we got back an ANA error, we know the controller is alive but not
  94          * ready to serve this namespace.  Kick of a re-read of the ANA
  95          * information page, and just try any other available path for now.
  96          */
  97         if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {
  98                 set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  99                 queue_work(nvme_wq, &ns->ctrl->ana_work);
 100         }
 101
 102         spin_lock_irqsave(&ns->head->requeue_lock, flags);
 103         for (bio = req->bio; bio; bio = bio->bi_next) {
 104                 bio_set_dev(bio, ns->head->disk->part0);
 105                 if (bio->bi_opf & REQ_POLLED) {
 106                         bio->bi_opf &= ~REQ_POLLED;
 107                         bio->bi_cookie = BLK_QC_T_NONE;
 108                 }
 109         }
 110         blk_steal_bios(&ns->head->requeue_list, req);
 111         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
 112
 113         blk_mq_end_request(req, 0);
 114         kblockd_schedule_work(&ns->head->requeue_work);
 115 }
 116
 117 void nvme_mpath_start_request(struct request *rq)
 118 {
 119         struct nvme_ns *ns = rq->q->queuedata;
 120         struct gendisk *disk = ns->head->disk;
 121
 122         if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 123                 return;
 124
 125         nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 126         nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 127                                                       jiffies);
 128 }
 129 EXPORT_SYMBOL_GPL(nvme_mpath_start_request);
 130
 131 void nvme_mpath_end_request(struct request *rq)
 132 {
 133         struct nvme_ns *ns = rq->q->queuedata;
 134
 135         if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 136                 return;
 137         bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 138                          blk_rq_bytes(rq) >> SECTOR_SHIFT,
 139                          nvme_req(rq)->start_time);
 140 }
 141
 142 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 143 {
 144         struct nvme_ns *ns;
 145
 146         down_read(&ctrl->namespaces_rwsem);
 147         list_for_each_entry(ns, &ctrl->namespaces, list) {
 148                 if (!ns->head->disk)
 149                         continue;
 150                 kblockd_schedule_work(&ns->head->requeue_work);
 151                 if (ctrl->state == NVME_CTRL_LIVE)
 152                         disk_uevent(ns->head->disk, KOBJ_CHANGE);
 153         }
 154         up_read(&ctrl->namespaces_rwsem);
 155 }
 156
 157 static const char *nvme_ana_state_names[] = {
 158         [0]                             = "invalid state",
 159         [NVME_ANA_OPTIMIZED]            = "optimized",
 160         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 161         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 162         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 163         [NVME_ANA_CHANGE]               = "change",
 164 };
 165
 166 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 167 {
 168         struct nvme_ns_head *head = ns->head;
 169         bool changed = false;
 170         int node;
 171
 172         if (!head)
 173                 goto out;
 174
 175         for_each_node(node) {
 176                 if (ns == rcu_access_pointer(head->current_path[node])) {
 177                         rcu_assign_pointer(head->current_path[node], NULL);
 178                         changed = true;
 179                 }
 180         }
 181 out:
 182         return changed;
 183 }
 184
 185 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 186 {
 187         struct nvme_ns *ns;
 188
 189         down_read(&ctrl->namespaces_rwsem);
 190         list_for_each_entry(ns, &ctrl->namespaces, list) {
 191                 nvme_mpath_clear_current_path(ns);
 192                 kblockd_schedule_work(&ns->head->requeue_work);
 193         }
 194         up_read(&ctrl->namespaces_rwsem);
 195 }
 196
 197 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
 198 {
 199         struct nvme_ns_head *head = ns->head;
 200         sector_t capacity = get_capacity(head->disk);
 201         int node;
 202         int srcu_idx;
 203
 204         srcu_idx = srcu_read_lock(&head->srcu);
 205         list_for_each_entry_rcu(ns, &head->list, siblings) {
 206                 if (capacity != get_capacity(ns->disk))
 207                         clear_bit(NVME_NS_READY, &ns->flags);
 208         }
 209         srcu_read_unlock(&head->srcu, srcu_idx);
 210
 211         for_each_node(node)
 212                 rcu_assign_pointer(head->current_path[node], NULL);
 213         kblockd_schedule_work(&head->requeue_work);
 214 }
 215
 216 static bool nvme_path_is_disabled(struct nvme_ns *ns)
 217 {
 218         /*
 219          * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should
 220          * still be able to complete assuming that the controller is connected.
 221          * Otherwise it will fail immediately and return to the requeue list.
 222          */
 223         if (ns->ctrl->state != NVME_CTRL_LIVE &&
 224             ns->ctrl->state != NVME_CTRL_DELETING)
 225                 return true;
 226         if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
 227             !test_bit(NVME_NS_READY, &ns->flags))
 228                 return true;
 229         return false;
 230 }
 231
 232 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 233 {
 234         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 235         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 236
 237         list_for_each_entry_rcu(ns, &head->list, siblings) {
 238                 if (nvme_path_is_disabled(ns))
 239                         continue;
 240
 241                 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
 242                         distance = node_distance(node, ns->ctrl->numa_node);
 243                 else
 244                         distance = LOCAL_DISTANCE;
 245
 246                 switch (ns->ana_state) {
 247                 case NVME_ANA_OPTIMIZED:
 248                         if (distance < found_distance) {
 249                                 found_distance = distance;
 250                                 found = ns;
 251                         }
 252                         break;
 253                 case NVME_ANA_NONOPTIMIZED:
 254                         if (distance < fallback_distance) {
 255                                 fallback_distance = distance;
 256                                 fallback = ns;
 257                         }
 258                         break;
 259                 default:
 260                         break;
 261                 }
 262         }
 263
 264         if (!found)
 265                 found = fallback;
 266         if (found)
 267                 rcu_assign_pointer(head->current_path[node], found);
 268         return found;
 269 }
 270
 271 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
 272                 struct nvme_ns *ns)
 273 {
 274         ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
 275                         siblings);
 276         if (ns)
 277                 return ns;
 278         return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
 279 }
 280
 281 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 282                 int node, struct nvme_ns *old)
 283 {
 284         struct nvme_ns *ns, *found = NULL;
 285
 286         if (list_is_singular(&head->list)) {
 287                 if (nvme_path_is_disabled(old))
 288                         return NULL;
 289                 return old;
 290         }
 291
 292         for (ns = nvme_next_ns(head, old);
 293              ns && ns != old;
 294              ns = nvme_next_ns(head, ns)) {
 295                 if (nvme_path_is_disabled(ns))
 296                         continue;
 297
 298                 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 299                         found = ns;
 300                         goto out;
 301                 }
 302                 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 303                         found = ns;
 304         }
 305
 306         /*
 307          * The loop above skips the current path for round-robin semantics.
 308          * Fall back to the current path if either:
 309          *  - no other optimized path found and current is optimized,
 310          *  - no other usable path found and current is usable.
 311          */
 312         if (!nvme_path_is_disabled(old) &&
 313             (old->ana_state == NVME_ANA_OPTIMIZED ||
 314              (!found && old->ana_state == NVME_ANA_NONOPTIMIZED)))
 315                 return old;
 316
 317         if (!found)
 318                 return NULL;
 319 out:
 320         rcu_assign_pointer(head->current_path[node], found);
 321         return found;
 322 }
 323
 324 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 325 {
 326         return ns->ctrl->state == NVME_CTRL_LIVE &&
 327                 ns->ana_state == NVME_ANA_OPTIMIZED;
 328 }
 329
 330 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 331 {
 332         int node = numa_node_id();
 333         struct nvme_ns *ns;
 334
 335         ns = srcu_dereference(head->current_path[node], &head->srcu);
 336         if (unlikely(!ns))
 337                 return __nvme_find_path(head, node);
 338
 339         if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR)
 340                 return nvme_round_robin_path(head, node, ns);
 341         if (unlikely(!nvme_path_is_optimized(ns)))
 342                 return __nvme_find_path(head, node);
 343         return ns;
 344 }
 345
 346 static bool nvme_available_path(struct nvme_ns_head *head)
 347 {
 348         struct nvme_ns *ns;
 349
 350         list_for_each_entry_rcu(ns, &head->list, siblings) {
 351                 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))
 352                         continue;
 353                 switch (ns->ctrl->state) {
 354                 case NVME_CTRL_LIVE:
 355                 case NVME_CTRL_RESETTING:
 356                 case NVME_CTRL_CONNECTING:
 357                         /* fallthru */
 358                         return true;
 359                 default:
 360                         break;
 361                 }
 362         }
 363         return false;
 364 }
 365
 366 static void nvme_ns_head_submit_bio(struct bio *bio)
 367 {
 368         struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
 369         struct device *dev = disk_to_dev(head->disk);
 370         struct nvme_ns *ns;
 371         int srcu_idx;
 372
 373         /*
 374          * The namespace might be going away and the bio might be moved to a
 375          * different queue via blk_steal_bios(), so we need to use the bio_split
 376          * pool from the original queue to allocate the bvecs from.
 377          */
 378         bio = bio_split_to_limits(bio);
 379         if (!bio)
 380                 return;
 381
 382         srcu_idx = srcu_read_lock(&head->srcu);
 383         ns = nvme_find_path(head);
 384         if (likely(ns)) {
 385                 bio_set_dev(bio, ns->disk->part0);
 386                 bio->bi_opf |= REQ_NVME_MPATH;
 387                 trace_block_bio_remap(bio, disk_devt(ns->head->disk),
 388                                       bio->bi_iter.bi_sector);
 389                 submit_bio_noacct(bio);
 390         } else if (nvme_available_path(head)) {
 391                 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 392
 393                 spin_lock_irq(&head->requeue_lock);
 394                 bio_list_add(&head->requeue_list, bio);
 395                 spin_unlock_irq(&head->requeue_lock);
 396         } else {
 397                 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 398
 399                 bio_io_error(bio);
 400         }
 401
 402         srcu_read_unlock(&head->srcu, srcu_idx);
 403 }
 404
 405 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
 406 {
 407         if (!nvme_tryget_ns_head(bdev->bd_disk->private_data))
 408                 return -ENXIO;
 409         return 0;
 410 }
 411
 412 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
 413 {
 414         nvme_put_ns_head(disk->private_data);
 415 }
 416
 417 #ifdef CONFIG_BLK_DEV_ZONED
 418 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector,
 419                 unsigned int nr_zones, report_zones_cb cb, void *data)
 420 {
 421         struct nvme_ns_head *head = disk->private_data;
 422         struct nvme_ns *ns;
 423         int srcu_idx, ret = -EWOULDBLOCK;
 424
 425         srcu_idx = srcu_read_lock(&head->srcu);
 426         ns = nvme_find_path(head);
 427         if (ns)
 428                 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);
 429         srcu_read_unlock(&head->srcu, srcu_idx);
 430         return ret;
 431 }
 432 #else
 433 #define nvme_ns_head_report_zones       NULL
 434 #endif /* CONFIG_BLK_DEV_ZONED */
 435
 436 const struct block_device_operations nvme_ns_head_ops = {
 437         .owner          = THIS_MODULE,
 438         .submit_bio     = nvme_ns_head_submit_bio,
 439         .open           = nvme_ns_head_open,
 440         .release        = nvme_ns_head_release,
 441         .ioctl          = nvme_ns_head_ioctl,
 442         .compat_ioctl   = blkdev_compat_ptr_ioctl,
 443         .getgeo         = nvme_getgeo,
 444         .report_zones   = nvme_ns_head_report_zones,
 445         .pr_ops         = &nvme_pr_ops,
 446 };
 447
 448 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev)
 449 {
 450         return container_of(cdev, struct nvme_ns_head, cdev);
 451 }
 452
 453 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file)
 454 {
 455         if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev)))
 456                 return -ENXIO;
 457         return 0;
 458 }
 459
 460 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file)
 461 {
 462         nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev));
 463         return 0;
 464 }
 465
 466 static const struct file_operations nvme_ns_head_chr_fops = {
 467         .owner          = THIS_MODULE,
 468         .open           = nvme_ns_head_chr_open,
 469         .release        = nvme_ns_head_chr_release,
 470         .unlocked_ioctl = nvme_ns_head_chr_ioctl,
 471         .compat_ioctl   = compat_ptr_ioctl,
 472         .uring_cmd      = nvme_ns_head_chr_uring_cmd,
 473         .uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll,
 474 };
 475
 476 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)
 477 {
 478         int ret;
 479
 480         head->cdev_device.parent = &head->subsys->dev;
 481         ret = dev_set_name(&head->cdev_device, "ng%dn%d",
 482                            head->subsys->instance, head->instance);
 483         if (ret)
 484                 return ret;
 485         ret = nvme_cdev_add(&head->cdev, &head->cdev_device,
 486                             &nvme_ns_head_chr_fops, THIS_MODULE);
 487         return ret;
 488 }
 489
 490 static void nvme_requeue_work(struct work_struct *work)
 491 {
 492         struct nvme_ns_head *head =
 493                 container_of(work, struct nvme_ns_head, requeue_work);
 494         struct bio *bio, *next;
 495
 496         spin_lock_irq(&head->requeue_lock);
 497         next = bio_list_get(&head->requeue_list);
 498         spin_unlock_irq(&head->requeue_lock);
 499
 500         while ((bio = next) != NULL) {
 501                 next = bio->bi_next;
 502                 bio->bi_next = NULL;
 503
 504                 submit_bio_noacct(bio);
 505         }
 506 }
 507
 508 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 509 {
 510         bool vwc = false;
 511
 512         mutex_init(&head->lock);
 513         bio_list_init(&head->requeue_list);
 514         spin_lock_init(&head->requeue_lock);
 515         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 516
 517         /*
 518          * Add a multipath node if the subsystems supports multiple controllers.
 519          * We also do this for private namespaces as the namespace sharing flag
 520          * could change after a rescan.
 521          */
 522         if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
 523             !nvme_is_unique_nsid(ctrl, head) || !multipath)
 524                 return 0;
 525
 526         head->disk = blk_alloc_disk(ctrl->numa_node);
 527         if (!head->disk)
 528                 return -ENOMEM;
 529         head->disk->fops = &nvme_ns_head_ops;
 530         head->disk->private_data = head;
 531         sprintf(head->disk->disk_name, "nvme%dn%d",
 532                         ctrl->subsys->instance, head->instance);
 533
 534         blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue);
 535         blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue);
 536         blk_queue_flag_set(QUEUE_FLAG_IO_STAT, head->disk->queue);
 537         /*
 538          * This assumes all controllers that refer to a namespace either
 539          * support poll queues or not.  That is not a strict guarantee,
 540          * but if the assumption is wrong the effect is only suboptimal
 541          * performance but not correctness problem.
 542          */
 543         if (ctrl->tagset->nr_maps > HCTX_TYPE_POLL &&
 544             ctrl->tagset->map[HCTX_TYPE_POLL].nr_queues)
 545                 blk_queue_flag_set(QUEUE_FLAG_POLL, head->disk->queue);
 546
 547         /* set to a default value of 512 until the disk is validated */
 548         blk_queue_logical_block_size(head->disk->queue, 512);
 549         blk_set_stacking_limits(&head->disk->queue->limits);
 550         blk_queue_dma_alignment(head->disk->queue, 3);
 551
 552         /* we need to propagate up the VMC settings */
 553         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 554                 vwc = true;
 555         blk_queue_write_cache(head->disk->queue, vwc, vwc);
 556         return 0;
 557 }
 558
 559 static void nvme_mpath_set_live(struct nvme_ns *ns)
 560 {
 561         struct nvme_ns_head *head = ns->head;
 562         int rc;
 563
 564         if (!head->disk)
 565                 return;
 566
 567         /*
 568          * test_and_set_bit() is used because it is protecting against two nvme
 569          * paths simultaneously calling device_add_disk() on the same namespace
 570          * head.
 571          */
 572         if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 573                 rc = device_add_disk(&head->subsys->dev, head->disk,
 574                                      nvme_ns_id_attr_groups);
 575                 if (rc) {
 576                         clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags);
 577                         return;
 578                 }
 579                 nvme_add_ns_head_cdev(head);
 580         }
 581
 582         mutex_lock(&head->lock);
 583         if (nvme_path_is_optimized(ns)) {
 584                 int node, srcu_idx;
 585
 586                 srcu_idx = srcu_read_lock(&head->srcu);
 587                 for_each_node(node)
 588                         __nvme_find_path(head, node);
 589                 srcu_read_unlock(&head->srcu, srcu_idx);
 590         }
 591         mutex_unlock(&head->lock);
 592
 593         synchronize_srcu(&head->srcu);
 594         kblockd_schedule_work(&head->requeue_work);
 595 }
 596
 597 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 598                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 599                         void *))
 600 {
 601         void *base = ctrl->ana_log_buf;
 602         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 603         int error, i;
 604
 605         lockdep_assert_held(&ctrl->ana_lock);
 606
 607         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 608                 struct nvme_ana_group_desc *desc = base + offset;
 609                 u32 nr_nsids;
 610                 size_t nsid_buf_size;
 611
 612                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 613                         return -EINVAL;
 614
 615                 nr_nsids = le32_to_cpu(desc->nnsids);
 616                 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids);
 617
 618                 if (WARN_ON_ONCE(desc->grpid == 0))
 619                         return -EINVAL;
 620                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 621                         return -EINVAL;
 622                 if (WARN_ON_ONCE(desc->state == 0))
 623                         return -EINVAL;
 624                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 625                         return -EINVAL;
 626
 627                 offset += sizeof(*desc);
 628                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 629                         return -EINVAL;
 630
 631                 error = cb(ctrl, desc, data);
 632                 if (error)
 633                         return error;
 634
 635                 offset += nsid_buf_size;
 636         }
 637
 638         return 0;
 639 }
 640
 641 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 642 {
 643         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 644 }
 645
 646 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 647                 struct nvme_ns *ns)
 648 {
 649         ns->ana_grpid = le32_to_cpu(desc->grpid);
 650         ns->ana_state = desc->state;
 651         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 652         /*
 653          * nvme_mpath_set_live() will trigger I/O to the multipath path device
 654          * and in turn to this path device.  However we cannot accept this I/O
 655          * if the controller is not live.  This may deadlock if called from
 656          * nvme_mpath_init_identify() and the ctrl will never complete
 657          * initialization, preventing I/O from completing.  For this case we
 658          * will reprocess the ANA log page in nvme_mpath_update() once the
 659          * controller is ready.
 660          */
 661         if (nvme_state_is_live(ns->ana_state) &&
 662             ns->ctrl->state == NVME_CTRL_LIVE)
 663                 nvme_mpath_set_live(ns);
 664 }
 665
 666 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 667                 struct nvme_ana_group_desc *desc, void *data)
 668 {
 669         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 670         unsigned *nr_change_groups = data;
 671         struct nvme_ns *ns;
 672
 673         dev_dbg(ctrl->device, "ANA group %d: %s.\n",
 674                         le32_to_cpu(desc->grpid),
 675                         nvme_ana_state_names[desc->state]);
 676
 677         if (desc->state == NVME_ANA_CHANGE)
 678                 (*nr_change_groups)++;
 679
 680         if (!nr_nsids)
 681                 return 0;
 682
 683         down_read(&ctrl->namespaces_rwsem);
 684         list_for_each_entry(ns, &ctrl->namespaces, list) {
 685                 unsigned nsid;
 686 again:
 687                 nsid = le32_to_cpu(desc->nsids[n]);
 688                 if (ns->head->ns_id < nsid)
 689                         continue;
 690                 if (ns->head->ns_id == nsid)
 691                         nvme_update_ns_ana_state(desc, ns);
 692                 if (++n == nr_nsids)
 693                         break;
 694                 if (ns->head->ns_id > nsid)
 695                         goto again;
 696         }
 697         up_read(&ctrl->namespaces_rwsem);
 698         return 0;
 699 }
 700
 701 static int nvme_read_ana_log(struct nvme_ctrl *ctrl)
 702 {
 703         u32 nr_change_groups = 0;
 704         int error;
 705
 706         mutex_lock(&ctrl->ana_lock);
 707         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM,
 708                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 709         if (error) {
 710                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 711                 goto out_unlock;
 712         }
 713
 714         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 715                         nvme_update_ana_state);
 716         if (error)
 717                 goto out_unlock;
 718
 719         /*
 720          * In theory we should have an ANATT timer per group as they might enter
 721          * the change state at different times.  But that is a lot of overhead
 722          * just to protect against a target that keeps entering new changes
 723          * states while never finishing previous ones.  But we'll still
 724          * eventually time out once all groups are in change state, so this
 725          * isn't a big deal.
 726          *
 727          * We also double the ANATT value to provide some slack for transports
 728          * or AEN processing overhead.
 729          */
 730         if (nr_change_groups)
 731                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 732         else
 733                 del_timer_sync(&ctrl->anatt_timer);
 734 out_unlock:
 735         mutex_unlock(&ctrl->ana_lock);
 736         return error;
 737 }
 738
 739 static void nvme_ana_work(struct work_struct *work)
 740 {
 741         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 742
 743         if (ctrl->state != NVME_CTRL_LIVE)
 744                 return;
 745
 746         nvme_read_ana_log(ctrl);
 747 }
 748
 749 void nvme_mpath_update(struct nvme_ctrl *ctrl)
 750 {
 751         u32 nr_change_groups = 0;
 752
 753         if (!ctrl->ana_log_buf)
 754                 return;
 755
 756         mutex_lock(&ctrl->ana_lock);
 757         nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state);
 758         mutex_unlock(&ctrl->ana_lock);
 759 }
 760
 761 static void nvme_anatt_timeout(struct timer_list *t)
 762 {
 763         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 764
 765         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 766         nvme_reset_ctrl(ctrl);
 767 }
 768
 769 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 770 {
 771         if (!nvme_ctrl_use_ana(ctrl))
 772                 return;
 773         del_timer_sync(&ctrl->anatt_timer);
 774         cancel_work_sync(&ctrl->ana_work);
 775 }
 776
 777 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store)  \
 778         struct device_attribute subsys_attr_##_name =   \
 779                 __ATTR(_name, _mode, _show, _store)
 780
 781 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 782                 struct device_attribute *attr, char *buf)
 783 {
 784         struct nvme_subsystem *subsys =
 785                 container_of(dev, struct nvme_subsystem, dev);
 786
 787         return sysfs_emit(buf, "%s\n",
 788                           nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 789 }
 790
 791 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 792                 struct device_attribute *attr, const char *buf, size_t count)
 793 {
 794         struct nvme_subsystem *subsys =
 795                 container_of(dev, struct nvme_subsystem, dev);
 796         int i;
 797
 798         for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 799                 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
 800                         WRITE_ONCE(subsys->iopolicy, i);
 801                         return count;
 802                 }
 803         }
 804
 805         return -EINVAL;
 806 }
 807 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
 808                       nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
 809
 810 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 811                 char *buf)
 812 {
 813         return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 814 }
 815 DEVICE_ATTR_RO(ana_grpid);
 816
 817 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 818                 char *buf)
 819 {
 820         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 821
 822         return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 823 }
 824 DEVICE_ATTR_RO(ana_state);
 825
 826 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
 827                 struct nvme_ana_group_desc *desc, void *data)
 828 {
 829         struct nvme_ana_group_desc *dst = data;
 830
 831         if (desc->grpid != dst->grpid)
 832                 return 0;
 833
 834         *dst = *desc;
 835         return -ENXIO; /* just break out of the loop */
 836 }
 837
 838 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)
 839 {
 840         if (nvme_ctrl_use_ana(ns->ctrl)) {
 841                 struct nvme_ana_group_desc desc = {
 842                         .grpid = anagrpid,
 843                         .state = 0,
 844                 };
 845
 846                 mutex_lock(&ns->ctrl->ana_lock);
 847                 ns->ana_grpid = le32_to_cpu(anagrpid);
 848                 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);
 849                 mutex_unlock(&ns->ctrl->ana_lock);
 850                 if (desc.state) {
 851                         /* found the group desc: update */
 852                         nvme_update_ns_ana_state(&desc, ns);
 853                 } else {
 854                         /* group desc not found: trigger a re-read */
 855                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
 856                         queue_work(nvme_wq, &ns->ctrl->ana_work);
 857                 }
 858         } else {
 859                 ns->ana_state = NVME_ANA_OPTIMIZED;
 860                 nvme_mpath_set_live(ns);
 861         }
 862
 863         if (blk_queue_stable_writes(ns->queue) && ns->head->disk)
 864                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
 865                                    ns->head->disk->queue);
 866 #ifdef CONFIG_BLK_DEV_ZONED
 867         if (blk_queue_is_zoned(ns->queue) && ns->head->disk)
 868                 ns->head->disk->nr_zones = ns->disk->nr_zones;
 869 #endif
 870 }
 871
 872 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head)
 873 {
 874         if (!head->disk)
 875                 return;
 876         kblockd_schedule_work(&head->requeue_work);
 877         if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 878                 nvme_cdev_del(&head->cdev, &head->cdev_device);
 879                 del_gendisk(head->disk);
 880         }
 881 }
 882
 883 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 884 {
 885         if (!head->disk)
 886                 return;
 887         /* make sure all pending bios are cleaned up */
 888         kblockd_schedule_work(&head->requeue_work);
 889         flush_work(&head->requeue_work);
 890         put_disk(head->disk);
 891 }
 892
 893 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
 894 {
 895         mutex_init(&ctrl->ana_lock);
 896         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 897         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 898 }
 899
 900 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 901 {
 902         size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT;
 903         size_t ana_log_size;
 904         int error = 0;
 905
 906         /* check if multipath is enabled and we have the capability */
 907         if (!multipath || !ctrl->subsys ||
 908             !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA))
 909                 return 0;
 910
 911         if (!ctrl->max_namespaces ||
 912             ctrl->max_namespaces > le32_to_cpu(id->nn)) {
 913                 dev_err(ctrl->device,
 914                         "Invalid MNAN value %u\n", ctrl->max_namespaces);
 915                 return -EINVAL;
 916         }
 917
 918         ctrl->anacap = id->anacap;
 919         ctrl->anatt = id->anatt;
 920         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 921         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 922
 923         ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 924                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) +
 925                 ctrl->max_namespaces * sizeof(__le32);
 926         if (ana_log_size > max_transfer_size) {
 927                 dev_err(ctrl->device,
 928                         "ANA log page size (%zd) larger than MDTS (%zd).\n",
 929                         ana_log_size, max_transfer_size);
 930                 dev_err(ctrl->device, "disabling ANA support.\n");
 931                 goto out_uninit;
 932         }
 933         if (ana_log_size > ctrl->ana_log_size) {
 934                 nvme_mpath_stop(ctrl);
 935                 nvme_mpath_uninit(ctrl);
 936                 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL);
 937                 if (!ctrl->ana_log_buf)
 938                         return -ENOMEM;
 939         }
 940         ctrl->ana_log_size = ana_log_size;
 941         error = nvme_read_ana_log(ctrl);
 942         if (error)
 943                 goto out_uninit;
 944         return 0;
 945
 946 out_uninit:
 947         nvme_mpath_uninit(ctrl);
 948         return error;
 949 }
 950
 951 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 952 {
 953         kvfree(ctrl->ana_log_buf);
 954         ctrl->ana_log_buf = NULL;
 955         ctrl->ana_log_size = 0;
 956 }