drivers/nvme/host/multipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2017-2018 Christoph Hellwig.
   4  */
   5
   6 #include <linux/moduleparam.h>
   7 #include <trace/events/block.h>
   8 #include "nvme.h"
   9
  10 static bool multipath = true;
  11 module_param(multipath, bool, 0444);
  12 MODULE_PARM_DESC(multipath,
  13         "turn on native support for multiple controllers per subsystem");
  14
  15 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
  16 {
  17         struct nvme_ns_head *h;
  18
  19         lockdep_assert_held(&subsys->lock);
  20         list_for_each_entry(h, &subsys->nsheads, entry)
  21                 if (h->disk)
  22                         blk_mq_unfreeze_queue(h->disk->queue);
  23 }
  24
  25 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
  26 {
  27         struct nvme_ns_head *h;
  28
  29         lockdep_assert_held(&subsys->lock);
  30         list_for_each_entry(h, &subsys->nsheads, entry)
  31                 if (h->disk)
  32                         blk_mq_freeze_queue_wait(h->disk->queue);
  33 }
  34
  35 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
  36 {
  37         struct nvme_ns_head *h;
  38
  39         lockdep_assert_held(&subsys->lock);
  40         list_for_each_entry(h, &subsys->nsheads, entry)
  41                 if (h->disk)
  42                         blk_freeze_queue_start(h->disk->queue);
  43 }
  44
  45 /*
  46  * If multipathing is enabled we need to always use the subsystem instance
  47  * number for numbering our devices to avoid conflicts between subsystems that
  48  * have multiple controllers and thus use the multipath-aware subsystem node
  49  * and those that have a single controller and use the controller node
  50  * directly.
  51  */
  52 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
  53                         struct nvme_ctrl *ctrl, int *flags)
  54 {
  55         if (!multipath) {
  56                 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
  57         } else if (ns->head->disk) {
  58                 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
  59                                 ctrl->instance, ns->head->instance);
  60                 *flags = GENHD_FL_HIDDEN;
  61         } else {
  62                 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
  63                                 ns->head->instance);
  64         }
  65 }
  66
  67 void nvme_failover_req(struct request *req)
  68 {
  69         struct nvme_ns *ns = req->q->queuedata;
  70         u16 status = nvme_req(req)->status;
  71         unsigned long flags;
  72
  73         spin_lock_irqsave(&ns->head->requeue_lock, flags);
  74         blk_steal_bios(&ns->head->requeue_list, req);
  75         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
  76         blk_mq_end_request(req, 0);
  77
  78         switch (status & 0x7ff) {
  79         case NVME_SC_ANA_TRANSITION:
  80         case NVME_SC_ANA_INACCESSIBLE:
  81         case NVME_SC_ANA_PERSISTENT_LOSS:
  82                 /*
  83                  * If we got back an ANA error we know the controller is alive,
  84                  * but not ready to serve this namespaces.  The spec suggests
  85                  * we should update our general state here, but due to the fact
  86                  * that the admin and I/O queues are not serialized that is
  87                  * fundamentally racy.  So instead just clear the current path,
  88                  * mark the the path as pending and kick of a re-read of the ANA
  89                  * log page ASAP.
  90                  */
  91                 nvme_mpath_clear_current_path(ns);
  92                 if (ns->ctrl->ana_log_buf) {
  93                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  94                         queue_work(nvme_wq, &ns->ctrl->ana_work);
  95                 }
  96                 break;
  97         case NVME_SC_HOST_PATH_ERROR:
  98                 /*
  99                  * Temporary transport disruption in talking to the controller.
 100                  * Try to send on a new path.
 101                  */
 102                 nvme_mpath_clear_current_path(ns);
 103                 break;
 104         default:
 105                 /*
 106                  * Reset the controller for any non-ANA error as we don't know
 107                  * what caused the error.
 108                  */
 109                 nvme_reset_ctrl(ns->ctrl);
 110                 break;
 111         }
 112
 113         kblockd_schedule_work(&ns->head->requeue_work);
 114 }
 115
 116 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 117 {
 118         struct nvme_ns *ns;
 119
 120         down_read(&ctrl->namespaces_rwsem);
 121         list_for_each_entry(ns, &ctrl->namespaces, list) {
 122                 if (ns->head->disk)
 123                         kblockd_schedule_work(&ns->head->requeue_work);
 124         }
 125         up_read(&ctrl->namespaces_rwsem);
 126 }
 127
 128 static const char *nvme_ana_state_names[] = {
 129         [0]                             = "invalid state",
 130         [NVME_ANA_OPTIMIZED]            = "optimized",
 131         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 132         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 133         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 134         [NVME_ANA_CHANGE]               = "change",
 135 };
 136
 137 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 138 {
 139         struct nvme_ns_head *head = ns->head;
 140         bool changed = false;
 141         int node;
 142
 143         if (!head)
 144                 goto out;
 145
 146         for_each_node(node) {
 147                 if (ns == rcu_access_pointer(head->current_path[node])) {
 148                         rcu_assign_pointer(head->current_path[node], NULL);
 149                         changed = true;
 150                 }
 151         }
 152 out:
 153         return changed;
 154 }
 155
 156 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 157 {
 158         struct nvme_ns *ns;
 159
 160         mutex_lock(&ctrl->scan_lock);
 161         list_for_each_entry(ns, &ctrl->namespaces, list)
 162                 if (nvme_mpath_clear_current_path(ns))
 163                         kblockd_schedule_work(&ns->head->requeue_work);
 164         mutex_unlock(&ctrl->scan_lock);
 165 }
 166
 167 static bool nvme_path_is_disabled(struct nvme_ns *ns)
 168 {
 169         return ns->ctrl->state != NVME_CTRL_LIVE ||
 170                 test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
 171                 test_bit(NVME_NS_REMOVING, &ns->flags);
 172 }
 173
 174 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 175 {
 176         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 177         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 178
 179         list_for_each_entry_rcu(ns, &head->list, siblings) {
 180                 if (nvme_path_is_disabled(ns))
 181                         continue;
 182
 183                 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
 184                         distance = node_distance(node, ns->ctrl->numa_node);
 185                 else
 186                         distance = LOCAL_DISTANCE;
 187
 188                 switch (ns->ana_state) {
 189                 case NVME_ANA_OPTIMIZED:
 190                         if (distance < found_distance) {
 191                                 found_distance = distance;
 192                                 found = ns;
 193                         }
 194                         break;
 195                 case NVME_ANA_NONOPTIMIZED:
 196                         if (distance < fallback_distance) {
 197                                 fallback_distance = distance;
 198                                 fallback = ns;
 199                         }
 200                         break;
 201                 default:
 202                         break;
 203                 }
 204         }
 205
 206         if (!found)
 207                 found = fallback;
 208         if (found)
 209                 rcu_assign_pointer(head->current_path[node], found);
 210         return found;
 211 }
 212
 213 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
 214                 struct nvme_ns *ns)
 215 {
 216         ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
 217                         siblings);
 218         if (ns)
 219                 return ns;
 220         return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
 221 }
 222
 223 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 224                 int node, struct nvme_ns *old)
 225 {
 226         struct nvme_ns *ns, *found, *fallback = NULL;
 227
 228         if (list_is_singular(&head->list)) {
 229                 if (nvme_path_is_disabled(old))
 230                         return NULL;
 231                 return old;
 232         }
 233
 234         for (ns = nvme_next_ns(head, old);
 235              ns != old;
 236              ns = nvme_next_ns(head, ns)) {
 237                 if (nvme_path_is_disabled(ns))
 238                         continue;
 239
 240                 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 241                         found = ns;
 242                         goto out;
 243                 }
 244                 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 245                         fallback = ns;
 246         }
 247
 248         if (!fallback)
 249                 return NULL;
 250         found = fallback;
 251 out:
 252         rcu_assign_pointer(head->current_path[node], found);
 253         return found;
 254 }
 255
 256 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 257 {
 258         return ns->ctrl->state == NVME_CTRL_LIVE &&
 259                 ns->ana_state == NVME_ANA_OPTIMIZED;
 260 }
 261
 262 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 263 {
 264         int node = numa_node_id();
 265         struct nvme_ns *ns;
 266
 267         ns = srcu_dereference(head->current_path[node], &head->srcu);
 268         if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR && ns)
 269                 ns = nvme_round_robin_path(head, node, ns);
 270         if (unlikely(!ns || !nvme_path_is_optimized(ns)))
 271                 ns = __nvme_find_path(head, node);
 272         return ns;
 273 }
 274
 275 static bool nvme_available_path(struct nvme_ns_head *head)
 276 {
 277         struct nvme_ns *ns;
 278
 279         list_for_each_entry_rcu(ns, &head->list, siblings) {
 280                 switch (ns->ctrl->state) {
 281                 case NVME_CTRL_LIVE:
 282                 case NVME_CTRL_RESETTING:
 283                 case NVME_CTRL_CONNECTING:
 284                         /* fallthru */
 285                         return true;
 286                 default:
 287                         break;
 288                 }
 289         }
 290         return false;
 291 }
 292
 293 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
 294                 struct bio *bio)
 295 {
 296         struct nvme_ns_head *head = q->queuedata;
 297         struct device *dev = disk_to_dev(head->disk);
 298         struct nvme_ns *ns;
 299         blk_qc_t ret = BLK_QC_T_NONE;
 300         int srcu_idx;
 301
 302         /*
 303          * The namespace might be going away and the bio might
 304          * be moved to a different queue via blk_steal_bios(),
 305          * so we need to use the bio_split pool from the original
 306          * queue to allocate the bvecs from.
 307          */
 308         blk_queue_split(q, &bio);
 309
 310         srcu_idx = srcu_read_lock(&head->srcu);
 311         ns = nvme_find_path(head);
 312         if (likely(ns)) {
 313                 bio->bi_disk = ns->disk;
 314                 bio->bi_opf |= REQ_NVME_MPATH;
 315                 trace_block_bio_remap(bio->bi_disk->queue, bio,
 316                                       disk_devt(ns->head->disk),
 317                                       bio->bi_iter.bi_sector);
 318                 ret = direct_make_request(bio);
 319         } else if (nvme_available_path(head)) {
 320                 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 321
 322                 spin_lock_irq(&head->requeue_lock);
 323                 bio_list_add(&head->requeue_list, bio);
 324                 spin_unlock_irq(&head->requeue_lock);
 325         } else {
 326                 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 327
 328                 bio->bi_status = BLK_STS_IOERR;
 329                 bio_endio(bio);
 330         }
 331
 332         srcu_read_unlock(&head->srcu, srcu_idx);
 333         return ret;
 334 }
 335
 336 static void nvme_requeue_work(struct work_struct *work)
 337 {
 338         struct nvme_ns_head *head =
 339                 container_of(work, struct nvme_ns_head, requeue_work);
 340         struct bio *bio, *next;
 341
 342         spin_lock_irq(&head->requeue_lock);
 343         next = bio_list_get(&head->requeue_list);
 344         spin_unlock_irq(&head->requeue_lock);
 345
 346         while ((bio = next) != NULL) {
 347                 next = bio->bi_next;
 348                 bio->bi_next = NULL;
 349
 350                 /*
 351                  * Reset disk to the mpath node and resubmit to select a new
 352                  * path.
 353                  */
 354                 bio->bi_disk = head->disk;
 355                 generic_make_request(bio);
 356         }
 357 }
 358
 359 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 360 {
 361         struct request_queue *q;
 362         bool vwc = false;
 363
 364         mutex_init(&head->lock);
 365         bio_list_init(&head->requeue_list);
 366         spin_lock_init(&head->requeue_lock);
 367         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 368
 369         /*
 370          * Add a multipath node if the subsystems supports multiple controllers.
 371          * We also do this for private namespaces as the namespace sharing data could
 372          * change after a rescan.
 373          */
 374         if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
 375                 return 0;
 376
 377         q = blk_alloc_queue_node(GFP_KERNEL, ctrl->numa_node);
 378         if (!q)
 379                 goto out;
 380         q->queuedata = head;
 381         blk_queue_make_request(q, nvme_ns_head_make_request);
 382         blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 383         /* set to a default value for 512 until disk is validated */
 384         blk_queue_logical_block_size(q, 512);
 385         blk_set_stacking_limits(&q->limits);
 386
 387         /* we need to propagate up the VMC settings */
 388         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 389                 vwc = true;
 390         blk_queue_write_cache(q, vwc, vwc);
 391
 392         head->disk = alloc_disk(0);
 393         if (!head->disk)
 394                 goto out_cleanup_queue;
 395         head->disk->fops = &nvme_ns_head_ops;
 396         head->disk->private_data = head;
 397         head->disk->queue = q;
 398         head->disk->flags = GENHD_FL_EXT_DEVT;
 399         sprintf(head->disk->disk_name, "nvme%dn%d",
 400                         ctrl->subsys->instance, head->instance);
 401         return 0;
 402
 403 out_cleanup_queue:
 404         blk_cleanup_queue(q);
 405 out:
 406         return -ENOMEM;
 407 }
 408
 409 static void nvme_mpath_set_live(struct nvme_ns *ns)
 410 {
 411         struct nvme_ns_head *head = ns->head;
 412
 413         lockdep_assert_held(&ns->head->lock);
 414
 415         if (!head->disk)
 416                 return;
 417
 418         if (!(head->disk->flags & GENHD_FL_UP))
 419                 device_add_disk(&head->subsys->dev, head->disk,
 420                                 nvme_ns_id_attr_groups);
 421
 422         if (nvme_path_is_optimized(ns)) {
 423                 int node, srcu_idx;
 424
 425                 srcu_idx = srcu_read_lock(&head->srcu);
 426                 for_each_node(node)
 427                         __nvme_find_path(head, node);
 428                 srcu_read_unlock(&head->srcu, srcu_idx);
 429         }
 430
 431         kblockd_schedule_work(&ns->head->requeue_work);
 432 }
 433
 434 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 435                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 436                         void *))
 437 {
 438         void *base = ctrl->ana_log_buf;
 439         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 440         int error, i;
 441
 442         lockdep_assert_held(&ctrl->ana_lock);
 443
 444         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 445                 struct nvme_ana_group_desc *desc = base + offset;
 446                 u32 nr_nsids = le32_to_cpu(desc->nnsids);
 447                 size_t nsid_buf_size = nr_nsids * sizeof(__le32);
 448
 449                 if (WARN_ON_ONCE(desc->grpid == 0))
 450                         return -EINVAL;
 451                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 452                         return -EINVAL;
 453                 if (WARN_ON_ONCE(desc->state == 0))
 454                         return -EINVAL;
 455                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 456                         return -EINVAL;
 457
 458                 offset += sizeof(*desc);
 459                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 460                         return -EINVAL;
 461
 462                 error = cb(ctrl, desc, data);
 463                 if (error)
 464                         return error;
 465
 466                 offset += nsid_buf_size;
 467                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 468                         return -EINVAL;
 469         }
 470
 471         return 0;
 472 }
 473
 474 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 475 {
 476         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 477 }
 478
 479 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 480                 struct nvme_ns *ns)
 481 {
 482         mutex_lock(&ns->head->lock);
 483         ns->ana_grpid = le32_to_cpu(desc->grpid);
 484         ns->ana_state = desc->state;
 485         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 486
 487         if (nvme_state_is_live(ns->ana_state))
 488                 nvme_mpath_set_live(ns);
 489         mutex_unlock(&ns->head->lock);
 490 }
 491
 492 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 493                 struct nvme_ana_group_desc *desc, void *data)
 494 {
 495         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 496         unsigned *nr_change_groups = data;
 497         struct nvme_ns *ns;
 498
 499         dev_dbg(ctrl->device, "ANA group %d: %s.\n",
 500                         le32_to_cpu(desc->grpid),
 501                         nvme_ana_state_names[desc->state]);
 502
 503         if (desc->state == NVME_ANA_CHANGE)
 504                 (*nr_change_groups)++;
 505
 506         if (!nr_nsids)
 507                 return 0;
 508
 509         down_write(&ctrl->namespaces_rwsem);
 510         list_for_each_entry(ns, &ctrl->namespaces, list) {
 511                 if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))
 512                         continue;
 513                 nvme_update_ns_ana_state(desc, ns);
 514                 if (++n == nr_nsids)
 515                         break;
 516         }
 517         up_write(&ctrl->namespaces_rwsem);
 518         WARN_ON_ONCE(n < nr_nsids);
 519         return 0;
 520 }
 521
 522 static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 523 {
 524         u32 nr_change_groups = 0;
 525         int error;
 526
 527         mutex_lock(&ctrl->ana_lock);
 528         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
 529                         groups_only ? NVME_ANA_LOG_RGO : 0,
 530                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 531         if (error) {
 532                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 533                 goto out_unlock;
 534         }
 535
 536         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 537                         nvme_update_ana_state);
 538         if (error)
 539                 goto out_unlock;
 540
 541         /*
 542          * In theory we should have an ANATT timer per group as they might enter
 543          * the change state at different times.  But that is a lot of overhead
 544          * just to protect against a target that keeps entering new changes
 545          * states while never finishing previous ones.  But we'll still
 546          * eventually time out once all groups are in change state, so this
 547          * isn't a big deal.
 548          *
 549          * We also double the ANATT value to provide some slack for transports
 550          * or AEN processing overhead.
 551          */
 552         if (nr_change_groups)
 553                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 554         else
 555                 del_timer_sync(&ctrl->anatt_timer);
 556 out_unlock:
 557         mutex_unlock(&ctrl->ana_lock);
 558         return error;
 559 }
 560
 561 static void nvme_ana_work(struct work_struct *work)
 562 {
 563         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 564
 565         nvme_read_ana_log(ctrl, false);
 566 }
 567
 568 static void nvme_anatt_timeout(struct timer_list *t)
 569 {
 570         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 571
 572         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 573         nvme_reset_ctrl(ctrl);
 574 }
 575
 576 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 577 {
 578         if (!nvme_ctrl_use_ana(ctrl))
 579                 return;
 580         del_timer_sync(&ctrl->anatt_timer);
 581         cancel_work_sync(&ctrl->ana_work);
 582 }
 583
 584 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store)  \
 585         struct device_attribute subsys_attr_##_name =   \
 586                 __ATTR(_name, _mode, _show, _store)
 587
 588 static const char *nvme_iopolicy_names[] = {
 589         [NVME_IOPOLICY_NUMA]    = "numa",
 590         [NVME_IOPOLICY_RR]      = "round-robin",
 591 };
 592
 593 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 594                 struct device_attribute *attr, char *buf)
 595 {
 596         struct nvme_subsystem *subsys =
 597                 container_of(dev, struct nvme_subsystem, dev);
 598
 599         return sprintf(buf, "%s\n",
 600                         nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 601 }
 602
 603 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 604                 struct device_attribute *attr, const char *buf, size_t count)
 605 {
 606         struct nvme_subsystem *subsys =
 607                 container_of(dev, struct nvme_subsystem, dev);
 608         int i;
 609
 610         for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 611                 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
 612                         WRITE_ONCE(subsys->iopolicy, i);
 613                         return count;
 614                 }
 615         }
 616
 617         return -EINVAL;
 618 }
 619 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
 620                       nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
 621
 622 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 623                 char *buf)
 624 {
 625         return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 626 }
 627 DEVICE_ATTR_RO(ana_grpid);
 628
 629 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 630                 char *buf)
 631 {
 632         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 633
 634         return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 635 }
 636 DEVICE_ATTR_RO(ana_state);
 637
 638 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
 639                 struct nvme_ana_group_desc *desc, void *data)
 640 {
 641         struct nvme_ns *ns = data;
 642
 643         if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
 644                 nvme_update_ns_ana_state(desc, ns);
 645                 return -ENXIO; /* just break out of the loop */
 646         }
 647
 648         return 0;
 649 }
 650
 651 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 652 {
 653         if (nvme_ctrl_use_ana(ns->ctrl)) {
 654                 mutex_lock(&ns->ctrl->ana_lock);
 655                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
 656                 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
 657                 mutex_unlock(&ns->ctrl->ana_lock);
 658         } else {
 659                 mutex_lock(&ns->head->lock);
 660                 ns->ana_state = NVME_ANA_OPTIMIZED;
 661                 nvme_mpath_set_live(ns);
 662                 mutex_unlock(&ns->head->lock);
 663         }
 664 }
 665
 666 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 667 {
 668         if (!head->disk)
 669                 return;
 670         if (head->disk->flags & GENHD_FL_UP)
 671                 del_gendisk(head->disk);
 672         blk_set_queue_dying(head->disk->queue);
 673         /* make sure all pending bios are cleaned up */
 674         kblockd_schedule_work(&head->requeue_work);
 675         flush_work(&head->requeue_work);
 676         blk_cleanup_queue(head->disk->queue);
 677         put_disk(head->disk);
 678 }
 679
 680 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 681 {
 682         int error;
 683
 684         /* check if multipath is enabled and we have the capability */
 685         if (!multipath || !ctrl->subsys || !(ctrl->subsys->cmic & (1 << 3)))
 686                 return 0;
 687
 688         ctrl->anacap = id->anacap;
 689         ctrl->anatt = id->anatt;
 690         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 691         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 692
 693         mutex_init(&ctrl->ana_lock);
 694         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 695         ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 696                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
 697         ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
 698
 699         if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
 700                 dev_err(ctrl->device,
 701                         "ANA log page size (%zd) larger than MDTS (%d).\n",
 702                         ctrl->ana_log_size,
 703                         ctrl->max_hw_sectors << SECTOR_SHIFT);
 704                 dev_err(ctrl->device, "disabling ANA support.\n");
 705                 return 0;
 706         }
 707
 708         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 709         ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
 710         if (!ctrl->ana_log_buf) {
 711                 error = -ENOMEM;
 712                 goto out;
 713         }
 714
 715         error = nvme_read_ana_log(ctrl, true);
 716         if (error)
 717                 goto out_free_ana_log_buf;
 718         return 0;
 719 out_free_ana_log_buf:
 720         kfree(ctrl->ana_log_buf);
 721         ctrl->ana_log_buf = NULL;
 722 out:
 723         return error;
 724 }
 725
 726 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 727 {
 728         kfree(ctrl->ana_log_buf);
 729         ctrl->ana_log_buf = NULL;
 730 }
 731