drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/compat.h>
  10 #include <linux/delay.h>
  11 #include <linux/errno.h>
  12 #include <linux/hdreg.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/backing-dev.h>
  16 #include <linux/list_sort.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <asm/unaligned.h>
  24
  25 #include "nvme.h"
  26 #include "fabrics.h"
  27
  28 #define CREATE_TRACE_POINTS
  29 #include "trace.h"
  30
  31 #define NVME_MINORS             (1U << MINORBITS)
  32
  33 unsigned int admin_timeout = 60;
  34 module_param(admin_timeout, uint, 0644);
  35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  36 EXPORT_SYMBOL_GPL(admin_timeout);
  37
  38 unsigned int nvme_io_timeout = 30;
  39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  42
  43 static unsigned char shutdown_timeout = 5;
  44 module_param(shutdown_timeout, byte, 0644);
  45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  46
  47 static u8 nvme_max_retries = 5;
  48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  50
  51 static unsigned long default_ps_max_latency_us = 100000;
  52 module_param(default_ps_max_latency_us, ulong, 0644);
  53 MODULE_PARM_DESC(default_ps_max_latency_us,
  54                  "max power saving latency for new devices; use PM QOS to change per device");
  55
  56 static bool force_apst;
  57 module_param(force_apst, bool, 0644);
  58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  59
  60 static bool streams;
  61 module_param(streams, bool, 0644);
  62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
  63
  64 /*
  65  * nvme_wq - hosts nvme related works that are not reset or delete
  66  * nvme_reset_wq - hosts nvme reset works
  67  * nvme_delete_wq - hosts nvme delete works
  68  *
  69  * nvme_wq will host works such as scan, aen handling, fw activation,
  70  * keep-alive, periodic reconnects etc. nvme_reset_wq
  71  * runs reset works which also flush works hosted on nvme_wq for
  72  * serialization purposes. nvme_delete_wq host controller deletion
  73  * works which flush reset works for serialization.
  74  */
  75 struct workqueue_struct *nvme_wq;
  76 EXPORT_SYMBOL_GPL(nvme_wq);
  77
  78 struct workqueue_struct *nvme_reset_wq;
  79 EXPORT_SYMBOL_GPL(nvme_reset_wq);
  80
  81 struct workqueue_struct *nvme_delete_wq;
  82 EXPORT_SYMBOL_GPL(nvme_delete_wq);
  83
  84 static LIST_HEAD(nvme_subsystems);
  85 static DEFINE_MUTEX(nvme_subsystems_lock);
  86
  87 static DEFINE_IDA(nvme_instance_ida);
  88 static dev_t nvme_chr_devt;
  89 static struct class *nvme_class;
  90 static struct class *nvme_subsys_class;
  91
  92 static int nvme_validate_ns(struct nvme_ns *ns);
  93 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
  94 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
  95                                            unsigned nsid);
  96
  97 static void nvme_update_bdev_size(struct gendisk *disk)
  98 {
  99         struct block_device *bdev = bdget_disk(disk, 0);
 100
 101         if (bdev) {
 102                 bd_set_nr_sectors(bdev, get_capacity(disk));
 103                 bdput(bdev);
 104         }
 105 }
 106
 107 /*
 108  * Prepare a queue for teardown.
 109  *
 110  * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
 111  * the capacity to 0 after that to avoid blocking dispatchers that may be
 112  * holding bd_butex.  This will end buffered writers dirtying pages that can't
 113  * be synced.
 114  */
 115 static void nvme_set_queue_dying(struct nvme_ns *ns)
 116 {
 117         if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
 118                 return;
 119
 120         blk_set_queue_dying(ns->queue);
 121         blk_mq_unquiesce_queue(ns->queue);
 122
 123         set_capacity(ns->disk, 0);
 124         nvme_update_bdev_size(ns->disk);
 125 }
 126
 127 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
 128 {
 129         /*
 130          * Only new queue scan work when admin and IO queues are both alive
 131          */
 132         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 133                 queue_work(nvme_wq, &ctrl->scan_work);
 134 }
 135
 136 /*
 137  * Use this function to proceed with scheduling reset_work for a controller
 138  * that had previously been set to the resetting state. This is intended for
 139  * code paths that can't be interrupted by other reset attempts. A hot removal
 140  * may prevent this from succeeding.
 141  */
 142 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 143 {
 144         if (ctrl->state != NVME_CTRL_RESETTING)
 145                 return -EBUSY;
 146         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 147                 return -EBUSY;
 148         return 0;
 149 }
 150 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 151
 152 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 153 {
 154         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 155                 return -EBUSY;
 156         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 157                 return -EBUSY;
 158         return 0;
 159 }
 160 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 161
 162 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 163 {
 164         int ret;
 165
 166         ret = nvme_reset_ctrl(ctrl);
 167         if (!ret) {
 168                 flush_work(&ctrl->reset_work);
 169                 if (ctrl->state != NVME_CTRL_LIVE)
 170                         ret = -ENETRESET;
 171         }
 172
 173         return ret;
 174 }
 175 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
 176
 177 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 178 {
 179         dev_info(ctrl->device,
 180                  "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
 181
 182         flush_work(&ctrl->reset_work);
 183         nvme_stop_ctrl(ctrl);
 184         nvme_remove_namespaces(ctrl);
 185         ctrl->ops->delete_ctrl(ctrl);
 186         nvme_uninit_ctrl(ctrl);
 187 }
 188
 189 static void nvme_delete_ctrl_work(struct work_struct *work)
 190 {
 191         struct nvme_ctrl *ctrl =
 192                 container_of(work, struct nvme_ctrl, delete_work);
 193
 194         nvme_do_delete_ctrl(ctrl);
 195 }
 196
 197 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 198 {
 199         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 200                 return -EBUSY;
 201         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 202                 return -EBUSY;
 203         return 0;
 204 }
 205 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 206
 207 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 208 {
 209         /*
 210          * Keep a reference until nvme_do_delete_ctrl() complete,
 211          * since ->delete_ctrl can free the controller.
 212          */
 213         nvme_get_ctrl(ctrl);
 214         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 215                 nvme_do_delete_ctrl(ctrl);
 216         nvme_put_ctrl(ctrl);
 217 }
 218
 219 static blk_status_t nvme_error_status(u16 status)
 220 {
 221         switch (status & 0x7ff) {
 222         case NVME_SC_SUCCESS:
 223                 return BLK_STS_OK;
 224         case NVME_SC_CAP_EXCEEDED:
 225                 return BLK_STS_NOSPC;
 226         case NVME_SC_LBA_RANGE:
 227         case NVME_SC_CMD_INTERRUPTED:
 228         case NVME_SC_NS_NOT_READY:
 229                 return BLK_STS_TARGET;
 230         case NVME_SC_BAD_ATTRIBUTES:
 231         case NVME_SC_ONCS_NOT_SUPPORTED:
 232         case NVME_SC_INVALID_OPCODE:
 233         case NVME_SC_INVALID_FIELD:
 234         case NVME_SC_INVALID_NS:
 235                 return BLK_STS_NOTSUPP;
 236         case NVME_SC_WRITE_FAULT:
 237         case NVME_SC_READ_ERROR:
 238         case NVME_SC_UNWRITTEN_BLOCK:
 239         case NVME_SC_ACCESS_DENIED:
 240         case NVME_SC_READ_ONLY:
 241         case NVME_SC_COMPARE_FAILED:
 242                 return BLK_STS_MEDIUM;
 243         case NVME_SC_GUARD_CHECK:
 244         case NVME_SC_APPTAG_CHECK:
 245         case NVME_SC_REFTAG_CHECK:
 246         case NVME_SC_INVALID_PI:
 247                 return BLK_STS_PROTECTION;
 248         case NVME_SC_RESERVATION_CONFLICT:
 249                 return BLK_STS_NEXUS;
 250         case NVME_SC_HOST_PATH_ERROR:
 251                 return BLK_STS_TRANSPORT;
 252         default:
 253                 return BLK_STS_IOERR;
 254         }
 255 }
 256
 257 static void nvme_retry_req(struct request *req)
 258 {
 259         struct nvme_ns *ns = req->q->queuedata;
 260         unsigned long delay = 0;
 261         u16 crd;
 262
 263         /* The mask and shift result must be <= 3 */
 264         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 265         if (ns && crd)
 266                 delay = ns->ctrl->crdt[crd - 1] * 100;
 267
 268         nvme_req(req)->retries++;
 269         blk_mq_requeue_request(req, false);
 270         blk_mq_delay_kick_requeue_list(req->q, delay);
 271 }
 272
 273 enum nvme_disposition {
 274         COMPLETE,
 275         RETRY,
 276         FAILOVER,
 277 };
 278
 279 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 280 {
 281         if (likely(nvme_req(req)->status == 0))
 282                 return COMPLETE;
 283
 284         if (blk_noretry_request(req) ||
 285             (nvme_req(req)->status & NVME_SC_DNR) ||
 286             nvme_req(req)->retries >= nvme_max_retries)
 287                 return COMPLETE;
 288
 289         if (req->cmd_flags & REQ_NVME_MPATH) {
 290                 if (nvme_is_path_error(nvme_req(req)->status) ||
 291                     blk_queue_dying(req->q))
 292                         return FAILOVER;
 293         } else {
 294                 if (blk_queue_dying(req->q))
 295                         return COMPLETE;
 296         }
 297
 298         return RETRY;
 299 }
 300
 301 static inline void nvme_end_req(struct request *req)
 302 {
 303         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 304
 305         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 306             req_op(req) == REQ_OP_ZONE_APPEND)
 307                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 308                         le64_to_cpu(nvme_req(req)->result.u64));
 309
 310         nvme_trace_bio_complete(req, status);
 311         blk_mq_end_request(req, status);
 312 }
 313
 314 void nvme_complete_rq(struct request *req)
 315 {
 316         trace_nvme_complete_rq(req);
 317         nvme_cleanup_cmd(req);
 318
 319         if (nvme_req(req)->ctrl->kas)
 320                 nvme_req(req)->ctrl->comp_seen = true;
 321
 322         switch (nvme_decide_disposition(req)) {
 323         case COMPLETE:
 324                 nvme_end_req(req);
 325                 return;
 326         case RETRY:
 327                 nvme_retry_req(req);
 328                 return;
 329         case FAILOVER:
 330                 nvme_failover_req(req);
 331                 return;
 332         }
 333 }
 334 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 335
 336 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
 337 {
 338         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 339                                 "Cancelling I/O %d", req->tag);
 340
 341         /* don't abort one completed request */
 342         if (blk_mq_request_completed(req))
 343                 return true;
 344
 345         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 346         blk_mq_complete_request(req);
 347         return true;
 348 }
 349 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 350
 351 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 352                 enum nvme_ctrl_state new_state)
 353 {
 354         enum nvme_ctrl_state old_state;
 355         unsigned long flags;
 356         bool changed = false;
 357
 358         spin_lock_irqsave(&ctrl->lock, flags);
 359
 360         old_state = ctrl->state;
 361         switch (new_state) {
 362         case NVME_CTRL_LIVE:
 363                 switch (old_state) {
 364                 case NVME_CTRL_NEW:
 365                 case NVME_CTRL_RESETTING:
 366                 case NVME_CTRL_CONNECTING:
 367                         changed = true;
 368                         fallthrough;
 369                 default:
 370                         break;
 371                 }
 372                 break;
 373         case NVME_CTRL_RESETTING:
 374                 switch (old_state) {
 375                 case NVME_CTRL_NEW:
 376                 case NVME_CTRL_LIVE:
 377                         changed = true;
 378                         fallthrough;
 379                 default:
 380                         break;
 381                 }
 382                 break;
 383         case NVME_CTRL_CONNECTING:
 384                 switch (old_state) {
 385                 case NVME_CTRL_NEW:
 386                 case NVME_CTRL_RESETTING:
 387                         changed = true;
 388                         fallthrough;
 389                 default:
 390                         break;
 391                 }
 392                 break;
 393         case NVME_CTRL_DELETING:
 394                 switch (old_state) {
 395                 case NVME_CTRL_LIVE:
 396                 case NVME_CTRL_RESETTING:
 397                 case NVME_CTRL_CONNECTING:
 398                         changed = true;
 399                         fallthrough;
 400                 default:
 401                         break;
 402                 }
 403                 break;
 404         case NVME_CTRL_DELETING_NOIO:
 405                 switch (old_state) {
 406                 case NVME_CTRL_DELETING:
 407                 case NVME_CTRL_DEAD:
 408                         changed = true;
 409                         fallthrough;
 410                 default:
 411                         break;
 412                 }
 413                 break;
 414         case NVME_CTRL_DEAD:
 415                 switch (old_state) {
 416                 case NVME_CTRL_DELETING:
 417                         changed = true;
 418                         fallthrough;
 419                 default:
 420                         break;
 421                 }
 422                 break;
 423         default:
 424                 break;
 425         }
 426
 427         if (changed) {
 428                 ctrl->state = new_state;
 429                 wake_up_all(&ctrl->state_wq);
 430         }
 431
 432         spin_unlock_irqrestore(&ctrl->lock, flags);
 433         if (changed && ctrl->state == NVME_CTRL_LIVE)
 434                 nvme_kick_requeue_lists(ctrl);
 435         return changed;
 436 }
 437 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 438
 439 /*
 440  * Returns true for sink states that can't ever transition back to live.
 441  */
 442 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 443 {
 444         switch (ctrl->state) {
 445         case NVME_CTRL_NEW:
 446         case NVME_CTRL_LIVE:
 447         case NVME_CTRL_RESETTING:
 448         case NVME_CTRL_CONNECTING:
 449                 return false;
 450         case NVME_CTRL_DELETING:
 451         case NVME_CTRL_DELETING_NOIO:
 452         case NVME_CTRL_DEAD:
 453                 return true;
 454         default:
 455                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 456                 return true;
 457         }
 458 }
 459
 460 /*
 461  * Waits for the controller state to be resetting, or returns false if it is
 462  * not possible to ever transition to that state.
 463  */
 464 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 465 {
 466         wait_event(ctrl->state_wq,
 467                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 468                    nvme_state_terminal(ctrl));
 469         return ctrl->state == NVME_CTRL_RESETTING;
 470 }
 471 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 472
 473 static void nvme_free_ns_head(struct kref *ref)
 474 {
 475         struct nvme_ns_head *head =
 476                 container_of(ref, struct nvme_ns_head, ref);
 477
 478         nvme_mpath_remove_disk(head);
 479         ida_simple_remove(&head->subsys->ns_ida, head->instance);
 480         cleanup_srcu_struct(&head->srcu);
 481         nvme_put_subsystem(head->subsys);
 482         kfree(head);
 483 }
 484
 485 static void nvme_put_ns_head(struct nvme_ns_head *head)
 486 {
 487         kref_put(&head->ref, nvme_free_ns_head);
 488 }
 489
 490 static void nvme_free_ns(struct kref *kref)
 491 {
 492         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 493
 494         if (ns->ndev)
 495                 nvme_nvm_unregister(ns);
 496
 497         put_disk(ns->disk);
 498         nvme_put_ns_head(ns->head);
 499         nvme_put_ctrl(ns->ctrl);
 500         kfree(ns);
 501 }
 502
 503 void nvme_put_ns(struct nvme_ns *ns)
 504 {
 505         kref_put(&ns->kref, nvme_free_ns);
 506 }
 507 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 508
 509 static inline void nvme_clear_nvme_request(struct request *req)
 510 {
 511         if (!(req->rq_flags & RQF_DONTPREP)) {
 512                 nvme_req(req)->retries = 0;
 513                 nvme_req(req)->flags = 0;
 514                 req->rq_flags |= RQF_DONTPREP;
 515         }
 516 }
 517
 518 struct request *nvme_alloc_request(struct request_queue *q,
 519                 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
 520 {
 521         unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
 522         struct request *req;
 523
 524         if (qid == NVME_QID_ANY) {
 525                 req = blk_mq_alloc_request(q, op, flags);
 526         } else {
 527                 req = blk_mq_alloc_request_hctx(q, op, flags,
 528                                 qid ? qid - 1 : 0);
 529         }
 530         if (IS_ERR(req))
 531                 return req;
 532
 533         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 534         nvme_clear_nvme_request(req);
 535         nvme_req(req)->cmd = cmd;
 536
 537         return req;
 538 }
 539 EXPORT_SYMBOL_GPL(nvme_alloc_request);
 540
 541 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
 542 {
 543         struct nvme_command c;
 544
 545         memset(&c, 0, sizeof(c));
 546
 547         c.directive.opcode = nvme_admin_directive_send;
 548         c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
 549         c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
 550         c.directive.dtype = NVME_DIR_IDENTIFY;
 551         c.directive.tdtype = NVME_DIR_STREAMS;
 552         c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
 553
 554         return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
 555 }
 556
 557 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
 558 {
 559         return nvme_toggle_streams(ctrl, false);
 560 }
 561
 562 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
 563 {
 564         return nvme_toggle_streams(ctrl, true);
 565 }
 566
 567 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
 568                                   struct streams_directive_params *s, u32 nsid)
 569 {
 570         struct nvme_command c;
 571
 572         memset(&c, 0, sizeof(c));
 573         memset(s, 0, sizeof(*s));
 574
 575         c.directive.opcode = nvme_admin_directive_recv;
 576         c.directive.nsid = cpu_to_le32(nsid);
 577         c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
 578         c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
 579         c.directive.dtype = NVME_DIR_STREAMS;
 580
 581         return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
 582 }
 583
 584 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
 585 {
 586         struct streams_directive_params s;
 587         int ret;
 588
 589         if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
 590                 return 0;
 591         if (!streams)
 592                 return 0;
 593
 594         ret = nvme_enable_streams(ctrl);
 595         if (ret)
 596                 return ret;
 597
 598         ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
 599         if (ret)
 600                 goto out_disable_stream;
 601
 602         ctrl->nssa = le16_to_cpu(s.nssa);
 603         if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
 604                 dev_info(ctrl->device, "too few streams (%u) available\n",
 605                                         ctrl->nssa);
 606                 goto out_disable_stream;
 607         }
 608
 609         ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
 610         dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
 611         return 0;
 612
 613 out_disable_stream:
 614         nvme_disable_streams(ctrl);
 615         return ret;
 616 }
 617
 618 /*
 619  * Check if 'req' has a write hint associated with it. If it does, assign
 620  * a valid namespace stream to the write.
 621  */
 622 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
 623                                      struct request *req, u16 *control,
 624                                      u32 *dsmgmt)
 625 {
 626         enum rw_hint streamid = req->write_hint;
 627
 628         if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
 629                 streamid = 0;
 630         else {
 631                 streamid--;
 632                 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
 633                         return;
 634
 635                 *control |= NVME_RW_DTYPE_STREAMS;
 636                 *dsmgmt |= streamid << 16;
 637         }
 638
 639         if (streamid < ARRAY_SIZE(req->q->write_hints))
 640                 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
 641 }
 642
 643 static void nvme_setup_passthrough(struct request *req,
 644                 struct nvme_command *cmd)
 645 {
 646         memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
 647         /* passthru commands should let the driver set the SGL flags */
 648         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 649 }
 650
 651 static inline void nvme_setup_flush(struct nvme_ns *ns,
 652                 struct nvme_command *cmnd)
 653 {
 654         cmnd->common.opcode = nvme_cmd_flush;
 655         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 656 }
 657
 658 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 659                 struct nvme_command *cmnd)
 660 {
 661         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 662         struct nvme_dsm_range *range;
 663         struct bio *bio;
 664
 665         /*
 666          * Some devices do not consider the DSM 'Number of Ranges' field when
 667          * determining how much data to DMA. Always allocate memory for maximum
 668          * number of segments to prevent device reading beyond end of buffer.
 669          */
 670         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 671
 672         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 673         if (!range) {
 674                 /*
 675                  * If we fail allocation our range, fallback to the controller
 676                  * discard page. If that's also busy, it's safe to return
 677                  * busy, as we know we can make progress once that's freed.
 678                  */
 679                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 680                         return BLK_STS_RESOURCE;
 681
 682                 range = page_address(ns->ctrl->discard_page);
 683         }
 684
 685         __rq_for_each_bio(bio, req) {
 686                 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 687                 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 688
 689                 if (n < segments) {
 690                         range[n].cattr = cpu_to_le32(0);
 691                         range[n].nlb = cpu_to_le32(nlb);
 692                         range[n].slba = cpu_to_le64(slba);
 693                 }
 694                 n++;
 695         }
 696
 697         if (WARN_ON_ONCE(n != segments)) {
 698                 if (virt_to_page(range) == ns->ctrl->discard_page)
 699                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 700                 else
 701                         kfree(range);
 702                 return BLK_STS_IOERR;
 703         }
 704
 705         cmnd->dsm.opcode = nvme_cmd_dsm;
 706         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 707         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 708         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 709
 710         req->special_vec.bv_page = virt_to_page(range);
 711         req->special_vec.bv_offset = offset_in_page(range);
 712         req->special_vec.bv_len = alloc_size;
 713         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 714
 715         return BLK_STS_OK;
 716 }
 717
 718 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 719                 struct request *req, struct nvme_command *cmnd)
 720 {
 721         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 722                 return nvme_setup_discard(ns, req, cmnd);
 723
 724         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 725         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 726         cmnd->write_zeroes.slba =
 727                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 728         cmnd->write_zeroes.length =
 729                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 730         cmnd->write_zeroes.control = 0;
 731         return BLK_STS_OK;
 732 }
 733
 734 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 735                 struct request *req, struct nvme_command *cmnd,
 736                 enum nvme_opcode op)
 737 {
 738         struct nvme_ctrl *ctrl = ns->ctrl;
 739         u16 control = 0;
 740         u32 dsmgmt = 0;
 741
 742         if (req->cmd_flags & REQ_FUA)
 743                 control |= NVME_RW_FUA;
 744         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 745                 control |= NVME_RW_LR;
 746
 747         if (req->cmd_flags & REQ_RAHEAD)
 748                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 749
 750         cmnd->rw.opcode = op;
 751         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 752         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 753         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 754
 755         if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
 756                 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
 757
 758         if (ns->ms) {
 759                 /*
 760                  * If formated with metadata, the block layer always provides a
 761                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 762                  * we enable the PRACT bit for protection information or set the
 763                  * namespace capacity to zero to prevent any I/O.
 764                  */
 765                 if (!blk_integrity_rq(req)) {
 766                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 767                                 return BLK_STS_NOTSUPP;
 768                         control |= NVME_RW_PRINFO_PRACT;
 769                 }
 770
 771                 switch (ns->pi_type) {
 772                 case NVME_NS_DPS_PI_TYPE3:
 773                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 774                         break;
 775                 case NVME_NS_DPS_PI_TYPE1:
 776                 case NVME_NS_DPS_PI_TYPE2:
 777                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 778                                         NVME_RW_PRINFO_PRCHK_REF;
 779                         if (op == nvme_cmd_zone_append)
 780                                 control |= NVME_RW_APPEND_PIREMAP;
 781                         cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 782                         break;
 783                 }
 784         }
 785
 786         cmnd->rw.control = cpu_to_le16(control);
 787         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 788         return 0;
 789 }
 790
 791 void nvme_cleanup_cmd(struct request *req)
 792 {
 793         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 794                 struct nvme_ns *ns = req->rq_disk->private_data;
 795                 struct page *page = req->special_vec.bv_page;
 796
 797                 if (page == ns->ctrl->discard_page)
 798                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 799                 else
 800                         kfree(page_address(page) + req->special_vec.bv_offset);
 801         }
 802 }
 803 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 804
 805 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 806                 struct nvme_command *cmd)
 807 {
 808         blk_status_t ret = BLK_STS_OK;
 809
 810         nvme_clear_nvme_request(req);
 811
 812         memset(cmd, 0, sizeof(*cmd));
 813         switch (req_op(req)) {
 814         case REQ_OP_DRV_IN:
 815         case REQ_OP_DRV_OUT:
 816                 nvme_setup_passthrough(req, cmd);
 817                 break;
 818         case REQ_OP_FLUSH:
 819                 nvme_setup_flush(ns, cmd);
 820                 break;
 821         case REQ_OP_ZONE_RESET_ALL:
 822         case REQ_OP_ZONE_RESET:
 823                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 824                 break;
 825         case REQ_OP_ZONE_OPEN:
 826                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 827                 break;
 828         case REQ_OP_ZONE_CLOSE:
 829                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 830                 break;
 831         case REQ_OP_ZONE_FINISH:
 832                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 833                 break;
 834         case REQ_OP_WRITE_ZEROES:
 835                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 836                 break;
 837         case REQ_OP_DISCARD:
 838                 ret = nvme_setup_discard(ns, req, cmd);
 839                 break;
 840         case REQ_OP_READ:
 841                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 842                 break;
 843         case REQ_OP_WRITE:
 844                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 845                 break;
 846         case REQ_OP_ZONE_APPEND:
 847                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 848                 break;
 849         default:
 850                 WARN_ON_ONCE(1);
 851                 return BLK_STS_IOERR;
 852         }
 853
 854         cmd->common.command_id = req->tag;
 855         trace_nvme_setup_cmd(req, cmd);
 856         return ret;
 857 }
 858 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
 859
 860 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
 861 {
 862         struct completion *waiting = rq->end_io_data;
 863
 864         rq->end_io_data = NULL;
 865         complete(waiting);
 866 }
 867
 868 static void nvme_execute_rq_polled(struct request_queue *q,
 869                 struct gendisk *bd_disk, struct request *rq, int at_head)
 870 {
 871         DECLARE_COMPLETION_ONSTACK(wait);
 872
 873         WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
 874
 875         rq->cmd_flags |= REQ_HIPRI;
 876         rq->end_io_data = &wait;
 877         blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
 878
 879         while (!completion_done(&wait)) {
 880                 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
 881                 cond_resched();
 882         }
 883 }
 884
 885 /*
 886  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 887  * if the result is positive, it's an NVM Express status code
 888  */
 889 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 890                 union nvme_result *result, void *buffer, unsigned bufflen,
 891                 unsigned timeout, int qid, int at_head,
 892                 blk_mq_req_flags_t flags, bool poll)
 893 {
 894         struct request *req;
 895         int ret;
 896
 897         req = nvme_alloc_request(q, cmd, flags, qid);
 898         if (IS_ERR(req))
 899                 return PTR_ERR(req);
 900
 901         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
 902
 903         if (buffer && bufflen) {
 904                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
 905                 if (ret)
 906                         goto out;
 907         }
 908
 909         if (poll)
 910                 nvme_execute_rq_polled(req->q, NULL, req, at_head);
 911         else
 912                 blk_execute_rq(req->q, NULL, req, at_head);
 913         if (result)
 914                 *result = nvme_req(req)->result;
 915         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 916                 ret = -EINTR;
 917         else
 918                 ret = nvme_req(req)->status;
 919  out:
 920         blk_mq_free_request(req);
 921         return ret;
 922 }
 923 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
 924
 925 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 926                 void *buffer, unsigned bufflen)
 927 {
 928         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
 929                         NVME_QID_ANY, 0, 0, false);
 930 }
 931 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 932
 933 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
 934                 unsigned len, u32 seed, bool write)
 935 {
 936         struct bio_integrity_payload *bip;
 937         int ret = -ENOMEM;
 938         void *buf;
 939
 940         buf = kmalloc(len, GFP_KERNEL);
 941         if (!buf)
 942                 goto out;
 943
 944         ret = -EFAULT;
 945         if (write && copy_from_user(buf, ubuf, len))
 946                 goto out_free_meta;
 947
 948         bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
 949         if (IS_ERR(bip)) {
 950                 ret = PTR_ERR(bip);
 951                 goto out_free_meta;
 952         }
 953
 954         bip->bip_iter.bi_size = len;
 955         bip->bip_iter.bi_sector = seed;
 956         ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
 957                         offset_in_page(buf));
 958         if (ret == len)
 959                 return buf;
 960         ret = -ENOMEM;
 961 out_free_meta:
 962         kfree(buf);
 963 out:
 964         return ERR_PTR(ret);
 965 }
 966
 967 static u32 nvme_known_admin_effects(u8 opcode)
 968 {
 969         switch (opcode) {
 970         case nvme_admin_format_nvm:
 971                 return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
 972                         NVME_CMD_EFFECTS_CSE_MASK;
 973         case nvme_admin_sanitize_nvm:
 974                 return NVME_CMD_EFFECTS_CSE_MASK;
 975         default:
 976                 break;
 977         }
 978         return 0;
 979 }
 980
 981 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
 982 {
 983         u32 effects = 0;
 984
 985         if (ns) {
 986                 if (ns->head->effects)
 987                         effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
 988                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
 989                         dev_warn(ctrl->device,
 990                                  "IO command:%02x has unhandled effects:%08x\n",
 991                                  opcode, effects);
 992                 return 0;
 993         }
 994
 995         if (ctrl->effects)
 996                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
 997         effects |= nvme_known_admin_effects(opcode);
 998
 999         return effects;
1000 }
1001 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1002
1003 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1004                                u8 opcode)
1005 {
1006         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1007
1008         /*
1009          * For simplicity, IO to all namespaces is quiesced even if the command
1010          * effects say only one namespace is affected.
1011          */
1012         if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1013                 mutex_lock(&ctrl->scan_lock);
1014                 mutex_lock(&ctrl->subsys->lock);
1015                 nvme_mpath_start_freeze(ctrl->subsys);
1016                 nvme_mpath_wait_freeze(ctrl->subsys);
1017                 nvme_start_freeze(ctrl);
1018                 nvme_wait_freeze(ctrl);
1019         }
1020         return effects;
1021 }
1022
1023 static void nvme_update_formats(struct nvme_ctrl *ctrl, u32 *effects)
1024 {
1025         struct nvme_ns *ns;
1026
1027         down_read(&ctrl->namespaces_rwsem);
1028         list_for_each_entry(ns, &ctrl->namespaces, list)
1029                 if (nvme_validate_ns(ns))
1030                         nvme_set_queue_dying(ns);
1031                 else if (blk_queue_is_zoned(ns->disk->queue)) {
1032                         /*
1033                          * IO commands are required to fully revalidate a zoned
1034                          * device. Force the command effects to trigger rescan
1035                          * work so report zones can run in a context with
1036                          * unfrozen IO queues.
1037                          */
1038                         *effects |= NVME_CMD_EFFECTS_NCC;
1039                 }
1040         up_read(&ctrl->namespaces_rwsem);
1041 }
1042
1043 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1044 {
1045         /*
1046          * Revalidate LBA changes prior to unfreezing. This is necessary to
1047          * prevent memory corruption if a logical block size was changed by
1048          * this command.
1049          */
1050         if (effects & NVME_CMD_EFFECTS_LBCC)
1051                 nvme_update_formats(ctrl, &effects);
1052         if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1053                 nvme_unfreeze(ctrl);
1054                 nvme_mpath_unfreeze(ctrl->subsys);
1055                 mutex_unlock(&ctrl->subsys->lock);
1056                 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1057                 mutex_unlock(&ctrl->scan_lock);
1058         }
1059         if (effects & NVME_CMD_EFFECTS_CCC)
1060                 nvme_init_identify(ctrl);
1061         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1062                 nvme_queue_scan(ctrl);
1063                 flush_work(&ctrl->scan_work);
1064         }
1065 }
1066
1067 void nvme_execute_passthru_rq(struct request *rq)
1068 {
1069         struct nvme_command *cmd = nvme_req(rq)->cmd;
1070         struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1071         struct nvme_ns *ns = rq->q->queuedata;
1072         struct gendisk *disk = ns ? ns->disk : NULL;
1073         u32 effects;
1074
1075         effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1076         blk_execute_rq(rq->q, disk, rq, 0);
1077         nvme_passthru_end(ctrl, effects);
1078 }
1079 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1080
1081 static int nvme_submit_user_cmd(struct request_queue *q,
1082                 struct nvme_command *cmd, void __user *ubuffer,
1083                 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
1084                 u32 meta_seed, u64 *result, unsigned timeout)
1085 {
1086         bool write = nvme_is_write(cmd);
1087         struct nvme_ns *ns = q->queuedata;
1088         struct gendisk *disk = ns ? ns->disk : NULL;
1089         struct request *req;
1090         struct bio *bio = NULL;
1091         void *meta = NULL;
1092         int ret;
1093
1094         req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
1095         if (IS_ERR(req))
1096                 return PTR_ERR(req);
1097
1098         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
1099         nvme_req(req)->flags |= NVME_REQ_USERCMD;
1100
1101         if (ubuffer && bufflen) {
1102                 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
1103                                 GFP_KERNEL);
1104                 if (ret)
1105                         goto out;
1106                 bio = req->bio;
1107                 bio->bi_disk = disk;
1108                 if (disk && meta_buffer && meta_len) {
1109                         meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
1110                                         meta_seed, write);
1111                         if (IS_ERR(meta)) {
1112                                 ret = PTR_ERR(meta);
1113                                 goto out_unmap;
1114                         }
1115                         req->cmd_flags |= REQ_INTEGRITY;
1116                 }
1117         }
1118
1119         nvme_execute_passthru_rq(req);
1120         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
1121                 ret = -EINTR;
1122         else
1123                 ret = nvme_req(req)->status;
1124         if (result)
1125                 *result = le64_to_cpu(nvme_req(req)->result.u64);
1126         if (meta && !ret && !write) {
1127                 if (copy_to_user(meta_buffer, meta, meta_len))
1128                         ret = -EFAULT;
1129         }
1130         kfree(meta);
1131  out_unmap:
1132         if (bio)
1133                 blk_rq_unmap_user(bio);
1134  out:
1135         blk_mq_free_request(req);
1136         return ret;
1137 }
1138
1139 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
1140 {
1141         struct nvme_ctrl *ctrl = rq->end_io_data;
1142         unsigned long flags;
1143         bool startka = false;
1144
1145         blk_mq_free_request(rq);
1146
1147         if (status) {
1148                 dev_err(ctrl->device,
1149                         "failed nvme_keep_alive_end_io error=%d\n",
1150                                 status);
1151                 return;
1152         }
1153
1154         ctrl->comp_seen = false;
1155         spin_lock_irqsave(&ctrl->lock, flags);
1156         if (ctrl->state == NVME_CTRL_LIVE ||
1157             ctrl->state == NVME_CTRL_CONNECTING)
1158                 startka = true;
1159         spin_unlock_irqrestore(&ctrl->lock, flags);
1160         if (startka)
1161                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1162 }
1163
1164 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
1165 {
1166         struct request *rq;
1167
1168         rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
1169                         NVME_QID_ANY);
1170         if (IS_ERR(rq))
1171                 return PTR_ERR(rq);
1172
1173         rq->timeout = ctrl->kato * HZ;
1174         rq->end_io_data = ctrl;
1175
1176         blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1177
1178         return 0;
1179 }
1180
1181 static void nvme_keep_alive_work(struct work_struct *work)
1182 {
1183         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1184                         struct nvme_ctrl, ka_work);
1185         bool comp_seen = ctrl->comp_seen;
1186
1187         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1188                 dev_dbg(ctrl->device,
1189                         "reschedule traffic based keep-alive timer\n");
1190                 ctrl->comp_seen = false;
1191                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1192                 return;
1193         }
1194
1195         if (nvme_keep_alive(ctrl)) {
1196                 /* allocation failure, reset the controller */
1197                 dev_err(ctrl->device, "keep-alive failed\n");
1198                 nvme_reset_ctrl(ctrl);
1199                 return;
1200         }
1201 }
1202
1203 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1204 {
1205         if (unlikely(ctrl->kato == 0))
1206                 return;
1207
1208         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1209 }
1210
1211 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1212 {
1213         if (unlikely(ctrl->kato == 0))
1214                 return;
1215
1216         cancel_delayed_work_sync(&ctrl->ka_work);
1217 }
1218 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1219
1220 /*
1221  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1222  * flag, thus sending any new CNS opcodes has a big chance of not working.
1223  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1224  * (but not for any later version).
1225  */
1226 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1227 {
1228         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1229                 return ctrl->vs < NVME_VS(1, 2, 0);
1230         return ctrl->vs < NVME_VS(1, 1, 0);
1231 }
1232
1233 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1234 {
1235         struct nvme_command c = { };
1236         int error;
1237
1238         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1239         c.identify.opcode = nvme_admin_identify;
1240         c.identify.cns = NVME_ID_CNS_CTRL;
1241
1242         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1243         if (!*id)
1244                 return -ENOMEM;
1245
1246         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1247                         sizeof(struct nvme_id_ctrl));
1248         if (error)
1249                 kfree(*id);
1250         return error;
1251 }
1252
1253 static bool nvme_multi_css(struct nvme_ctrl *ctrl)
1254 {
1255         return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
1256 }
1257
1258 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1259                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1260 {
1261         const char *warn_str = "ctrl returned bogus length:";
1262         void *data = cur;
1263
1264         switch (cur->nidt) {
1265         case NVME_NIDT_EUI64:
1266                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1267                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1268                                  warn_str, cur->nidl);
1269                         return -1;
1270                 }
1271                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1272                 return NVME_NIDT_EUI64_LEN;
1273         case NVME_NIDT_NGUID:
1274                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1275                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1276                                  warn_str, cur->nidl);
1277                         return -1;
1278                 }
1279                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1280                 return NVME_NIDT_NGUID_LEN;
1281         case NVME_NIDT_UUID:
1282                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1283                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1284                                  warn_str, cur->nidl);
1285                         return -1;
1286                 }
1287                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1288                 return NVME_NIDT_UUID_LEN;
1289         case NVME_NIDT_CSI:
1290                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1291                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1292                                  warn_str, cur->nidl);
1293                         return -1;
1294                 }
1295                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1296                 *csi_seen = true;
1297                 return NVME_NIDT_CSI_LEN;
1298         default:
1299                 /* Skip unknown types */
1300                 return cur->nidl;
1301         }
1302 }
1303
1304 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1305                 struct nvme_ns_ids *ids)
1306 {
1307         struct nvme_command c = { };
1308         bool csi_seen = false;
1309         int status, pos, len;
1310         void *data;
1311
1312         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1313                 return 0;
1314
1315         c.identify.opcode = nvme_admin_identify;
1316         c.identify.nsid = cpu_to_le32(nsid);
1317         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1318
1319         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1320         if (!data)
1321                 return -ENOMEM;
1322
1323         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1324                                       NVME_IDENTIFY_DATA_SIZE);
1325         if (status) {
1326                 dev_warn(ctrl->device,
1327                         "Identify Descriptors failed (%d)\n", status);
1328                 goto free_data;
1329         }
1330
1331         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1332                 struct nvme_ns_id_desc *cur = data + pos;
1333
1334                 if (cur->nidl == 0)
1335                         break;
1336
1337                 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
1338                 if (len < 0)
1339                         break;
1340
1341                 len += sizeof(*cur);
1342         }
1343
1344         if (nvme_multi_css(ctrl) && !csi_seen) {
1345                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1346                          nsid);
1347                 status = -EINVAL;
1348         }
1349
1350 free_data:
1351         kfree(data);
1352         return status;
1353 }
1354
1355 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
1356 {
1357         struct nvme_command c = { };
1358
1359         c.identify.opcode = nvme_admin_identify;
1360         c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
1361         c.identify.nsid = cpu_to_le32(nsid);
1362         return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
1363                                     NVME_IDENTIFY_DATA_SIZE);
1364 }
1365
1366 static int nvme_identify_ns(struct nvme_ctrl *ctrl,
1367                 unsigned nsid, struct nvme_id_ns **id)
1368 {
1369         struct nvme_command c = { };
1370         int error;
1371
1372         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1373         c.identify.opcode = nvme_admin_identify;
1374         c.identify.nsid = cpu_to_le32(nsid);
1375         c.identify.cns = NVME_ID_CNS_NS;
1376
1377         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1378         if (!*id)
1379                 return -ENOMEM;
1380
1381         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1382         if (error) {
1383                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1384                 goto out_free_id;
1385         }
1386
1387         error = -ENODEV;
1388         if ((*id)->ncap == 0) /* namespace not allocated or attached */
1389                 goto out_free_id;
1390         return 0;
1391
1392 out_free_id:
1393         kfree(*id);
1394         return error;
1395 }
1396
1397 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1398                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1399 {
1400         union nvme_result res = { 0 };
1401         struct nvme_command c;
1402         int ret;
1403
1404         memset(&c, 0, sizeof(c));
1405         c.features.opcode = op;
1406         c.features.fid = cpu_to_le32(fid);
1407         c.features.dword11 = cpu_to_le32(dword11);
1408
1409         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1410                         buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1411         if (ret >= 0 && result)
1412                 *result = le32_to_cpu(res.u32);
1413         return ret;
1414 }
1415
1416 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1417                       unsigned int dword11, void *buffer, size_t buflen,
1418                       u32 *result)
1419 {
1420         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1421                              buflen, result);
1422 }
1423 EXPORT_SYMBOL_GPL(nvme_set_features);
1424
1425 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1426                       unsigned int dword11, void *buffer, size_t buflen,
1427                       u32 *result)
1428 {
1429         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1430                              buflen, result);
1431 }
1432 EXPORT_SYMBOL_GPL(nvme_get_features);
1433
1434 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1435 {
1436         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1437         u32 result;
1438         int status, nr_io_queues;
1439
1440         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1441                         &result);
1442         if (status < 0)
1443                 return status;
1444
1445         /*
1446          * Degraded controllers might return an error when setting the queue
1447          * count.  We still want to be able to bring them online and offer
1448          * access to the admin queue, as that might be only way to fix them up.
1449          */
1450         if (status > 0) {
1451                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1452                 *count = 0;
1453         } else {
1454                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1455                 *count = min(*count, nr_io_queues);
1456         }
1457
1458         return 0;
1459 }
1460 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1461
1462 #define NVME_AEN_SUPPORTED \
1463         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1464          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1465
1466 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1467 {
1468         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1469         int status;
1470
1471         if (!supported_aens)
1472                 return;
1473
1474         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1475                         NULL, 0, &result);
1476         if (status)
1477                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1478                          supported_aens);
1479
1480         queue_work(nvme_wq, &ctrl->async_event_work);
1481 }
1482
1483 /*
1484  * Convert integer values from ioctl structures to user pointers, silently
1485  * ignoring the upper bits in the compat case to match behaviour of 32-bit
1486  * kernels.
1487  */
1488 static void __user *nvme_to_user_ptr(uintptr_t ptrval)
1489 {
1490         if (in_compat_syscall())
1491                 ptrval = (compat_uptr_t)ptrval;
1492         return (void __user *)ptrval;
1493 }
1494
1495 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1496 {
1497         struct nvme_user_io io;
1498         struct nvme_command c;
1499         unsigned length, meta_len;
1500         void __user *metadata;
1501
1502         if (copy_from_user(&io, uio, sizeof(io)))
1503                 return -EFAULT;
1504         if (io.flags)
1505                 return -EINVAL;
1506
1507         switch (io.opcode) {
1508         case nvme_cmd_write:
1509         case nvme_cmd_read:
1510         case nvme_cmd_compare:
1511                 break;
1512         default:
1513                 return -EINVAL;
1514         }
1515
1516         length = (io.nblocks + 1) << ns->lba_shift;
1517         meta_len = (io.nblocks + 1) * ns->ms;
1518         metadata = nvme_to_user_ptr(io.metadata);
1519
1520         if (ns->features & NVME_NS_EXT_LBAS) {
1521                 length += meta_len;
1522                 meta_len = 0;
1523         } else if (meta_len) {
1524                 if ((io.metadata & 3) || !io.metadata)
1525                         return -EINVAL;
1526         }
1527
1528         memset(&c, 0, sizeof(c));
1529         c.rw.opcode = io.opcode;
1530         c.rw.flags = io.flags;
1531         c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1532         c.rw.slba = cpu_to_le64(io.slba);
1533         c.rw.length = cpu_to_le16(io.nblocks);
1534         c.rw.control = cpu_to_le16(io.control);
1535         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1536         c.rw.reftag = cpu_to_le32(io.reftag);
1537         c.rw.apptag = cpu_to_le16(io.apptag);
1538         c.rw.appmask = cpu_to_le16(io.appmask);
1539
1540         return nvme_submit_user_cmd(ns->queue, &c,
1541                         nvme_to_user_ptr(io.addr), length,
1542                         metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1543 }
1544
1545 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1546                         struct nvme_passthru_cmd __user *ucmd)
1547 {
1548         struct nvme_passthru_cmd cmd;
1549         struct nvme_command c;
1550         unsigned timeout = 0;
1551         u64 result;
1552         int status;
1553
1554         if (!capable(CAP_SYS_ADMIN))
1555                 return -EACCES;
1556         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1557                 return -EFAULT;
1558         if (cmd.flags)
1559                 return -EINVAL;
1560
1561         memset(&c, 0, sizeof(c));
1562         c.common.opcode = cmd.opcode;
1563         c.common.flags = cmd.flags;
1564         c.common.nsid = cpu_to_le32(cmd.nsid);
1565         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1566         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1567         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1568         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1569         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1570         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1571         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1572         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1573
1574         if (cmd.timeout_ms)
1575                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1576
1577         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1578                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1579                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1580                         0, &result, timeout);
1581
1582         if (status >= 0) {
1583                 if (put_user(result, &ucmd->result))
1584                         return -EFAULT;
1585         }
1586
1587         return status;
1588 }
1589
1590 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1591                         struct nvme_passthru_cmd64 __user *ucmd)
1592 {
1593         struct nvme_passthru_cmd64 cmd;
1594         struct nvme_command c;
1595         unsigned timeout = 0;
1596         int status;
1597
1598         if (!capable(CAP_SYS_ADMIN))
1599                 return -EACCES;
1600         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1601                 return -EFAULT;
1602         if (cmd.flags)
1603                 return -EINVAL;
1604
1605         memset(&c, 0, sizeof(c));
1606         c.common.opcode = cmd.opcode;
1607         c.common.flags = cmd.flags;
1608         c.common.nsid = cpu_to_le32(cmd.nsid);
1609         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1610         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1611         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1612         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1613         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1614         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1615         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1616         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1617
1618         if (cmd.timeout_ms)
1619                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1620
1621         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1622                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1623                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1624                         0, &cmd.result, timeout);
1625
1626         if (status >= 0) {
1627                 if (put_user(cmd.result, &ucmd->result))
1628                         return -EFAULT;
1629         }
1630
1631         return status;
1632 }
1633
1634 /*
1635  * Issue ioctl requests on the first available path.  Note that unlike normal
1636  * block layer requests we will not retry failed request on another controller.
1637  */
1638 struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1639                 struct nvme_ns_head **head, int *srcu_idx)
1640 {
1641 #ifdef CONFIG_NVME_MULTIPATH
1642         if (disk->fops == &nvme_ns_head_ops) {
1643                 struct nvme_ns *ns;
1644
1645                 *head = disk->private_data;
1646                 *srcu_idx = srcu_read_lock(&(*head)->srcu);
1647                 ns = nvme_find_path(*head);
1648                 if (!ns)
1649                         srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1650                 return ns;
1651         }
1652 #endif
1653         *head = NULL;
1654         *srcu_idx = -1;
1655         return disk->private_data;
1656 }
1657
1658 void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1659 {
1660         if (head)
1661                 srcu_read_unlock(&head->srcu, idx);
1662 }
1663
1664 static bool is_ctrl_ioctl(unsigned int cmd)
1665 {
1666         if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1667                 return true;
1668         if (is_sed_ioctl(cmd))
1669                 return true;
1670         return false;
1671 }
1672
1673 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1674                                   void __user *argp,
1675                                   struct nvme_ns_head *head,
1676                                   int srcu_idx)
1677 {
1678         struct nvme_ctrl *ctrl = ns->ctrl;
1679         int ret;
1680
1681         nvme_get_ctrl(ns->ctrl);
1682         nvme_put_ns_from_disk(head, srcu_idx);
1683
1684         switch (cmd) {
1685         case NVME_IOCTL_ADMIN_CMD:
1686                 ret = nvme_user_cmd(ctrl, NULL, argp);
1687                 break;
1688         case NVME_IOCTL_ADMIN64_CMD:
1689                 ret = nvme_user_cmd64(ctrl, NULL, argp);
1690                 break;
1691         default:
1692                 ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1693                 break;
1694         }
1695         nvme_put_ctrl(ctrl);
1696         return ret;
1697 }
1698
1699 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1700                 unsigned int cmd, unsigned long arg)
1701 {
1702         struct nvme_ns_head *head = NULL;
1703         void __user *argp = (void __user *)arg;
1704         struct nvme_ns *ns;
1705         int srcu_idx, ret;
1706
1707         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1708         if (unlikely(!ns))
1709                 return -EWOULDBLOCK;
1710
1711         /*
1712          * Handle ioctls that apply to the controller instead of the namespace
1713          * seperately and drop the ns SRCU reference early.  This avoids a
1714          * deadlock when deleting namespaces using the passthrough interface.
1715          */
1716         if (is_ctrl_ioctl(cmd))
1717                 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1718
1719         switch (cmd) {
1720         case NVME_IOCTL_ID:
1721                 force_successful_syscall_return();
1722                 ret = ns->head->ns_id;
1723                 break;
1724         case NVME_IOCTL_IO_CMD:
1725                 ret = nvme_user_cmd(ns->ctrl, ns, argp);
1726                 break;
1727         case NVME_IOCTL_SUBMIT_IO:
1728                 ret = nvme_submit_io(ns, argp);
1729                 break;
1730         case NVME_IOCTL_IO64_CMD:
1731                 ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1732                 break;
1733         default:
1734                 if (ns->ndev)
1735                         ret = nvme_nvm_ioctl(ns, cmd, arg);
1736                 else
1737                         ret = -ENOTTY;
1738         }
1739
1740         nvme_put_ns_from_disk(head, srcu_idx);
1741         return ret;
1742 }
1743
1744 #ifdef CONFIG_COMPAT
1745 struct nvme_user_io32 {
1746         __u8    opcode;
1747         __u8    flags;
1748         __u16   control;
1749         __u16   nblocks;
1750         __u16   rsvd;
1751         __u64   metadata;
1752         __u64   addr;
1753         __u64   slba;
1754         __u32   dsmgmt;
1755         __u32   reftag;
1756         __u16   apptag;
1757         __u16   appmask;
1758 } __attribute__((__packed__));
1759
1760 #define NVME_IOCTL_SUBMIT_IO32  _IOW('N', 0x42, struct nvme_user_io32)
1761
1762 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1763                 unsigned int cmd, unsigned long arg)
1764 {
1765         /*
1766          * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
1767          * between 32 bit programs and 64 bit kernel.
1768          * The cause is that the results of sizeof(struct nvme_user_io),
1769          * which is used to define NVME_IOCTL_SUBMIT_IO,
1770          * are not same between 32 bit compiler and 64 bit compiler.
1771          * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
1772          * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
1773          * Other IOCTL numbers are same between 32 bit and 64 bit.
1774          * So there is nothing to do regarding to other IOCTL numbers.
1775          */
1776         if (cmd == NVME_IOCTL_SUBMIT_IO32)
1777                 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
1778
1779         return nvme_ioctl(bdev, mode, cmd, arg);
1780 }
1781 #else
1782 #define nvme_compat_ioctl       NULL
1783 #endif /* CONFIG_COMPAT */
1784
1785 static int nvme_open(struct block_device *bdev, fmode_t mode)
1786 {
1787         struct nvme_ns *ns = bdev->bd_disk->private_data;
1788
1789 #ifdef CONFIG_NVME_MULTIPATH
1790         /* should never be called due to GENHD_FL_HIDDEN */
1791         if (WARN_ON_ONCE(ns->head->disk))
1792                 goto fail;
1793 #endif
1794         if (!kref_get_unless_zero(&ns->kref))
1795                 goto fail;
1796         if (!try_module_get(ns->ctrl->ops->module))
1797                 goto fail_put_ns;
1798
1799         return 0;
1800
1801 fail_put_ns:
1802         nvme_put_ns(ns);
1803 fail:
1804         return -ENXIO;
1805 }
1806
1807 static void nvme_release(struct gendisk *disk, fmode_t mode)
1808 {
1809         struct nvme_ns *ns = disk->private_data;
1810
1811         module_put(ns->ctrl->ops->module);
1812         nvme_put_ns(ns);
1813 }
1814
1815 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1816 {
1817         /* some standard values */
1818         geo->heads = 1 << 6;
1819         geo->sectors = 1 << 5;
1820         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1821         return 0;
1822 }
1823
1824 #ifdef CONFIG_BLK_DEV_INTEGRITY
1825 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1826                                 u32 max_integrity_segments)
1827 {
1828         struct blk_integrity integrity;
1829
1830         memset(&integrity, 0, sizeof(integrity));
1831         switch (pi_type) {
1832         case NVME_NS_DPS_PI_TYPE3:
1833                 integrity.profile = &t10_pi_type3_crc;
1834                 integrity.tag_size = sizeof(u16) + sizeof(u32);
1835                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1836                 break;
1837         case NVME_NS_DPS_PI_TYPE1:
1838         case NVME_NS_DPS_PI_TYPE2:
1839                 integrity.profile = &t10_pi_type1_crc;
1840                 integrity.tag_size = sizeof(u16);
1841                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1842                 break;
1843         default:
1844                 integrity.profile = NULL;
1845                 break;
1846         }
1847         integrity.tuple_size = ms;
1848         blk_integrity_register(disk, &integrity);
1849         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1850 }
1851 #else
1852 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1853                                 u32 max_integrity_segments)
1854 {
1855 }
1856 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1857
1858 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1859 {
1860         struct nvme_ctrl *ctrl = ns->ctrl;
1861         struct request_queue *queue = disk->queue;
1862         u32 size = queue_logical_block_size(queue);
1863
1864         if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1865                 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1866                 return;
1867         }
1868
1869         if (ctrl->nr_streams && ns->sws && ns->sgs)
1870                 size *= ns->sws * ns->sgs;
1871
1872         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1873                         NVME_DSM_MAX_RANGES);
1874
1875         queue->limits.discard_alignment = 0;
1876         queue->limits.discard_granularity = size;
1877
1878         /* If discard is already enabled, don't reset queue limits */
1879         if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1880                 return;
1881
1882         blk_queue_max_discard_sectors(queue, UINT_MAX);
1883         blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1884
1885         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1886                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1887 }
1888
1889 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
1890 {
1891         u64 max_blocks;
1892
1893         if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1894             (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1895                 return;
1896         /*
1897          * Even though NVMe spec explicitly states that MDTS is not
1898          * applicable to the write-zeroes:- "The restriction does not apply to
1899          * commands that do not transfer data between the host and the
1900          * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1901          * In order to be more cautious use controller's max_hw_sectors value
1902          * to configure the maximum sectors for the write-zeroes which is
1903          * configured based on the controller's MDTS field in the
1904          * nvme_init_identify() if available.
1905          */
1906         if (ns->ctrl->max_hw_sectors == UINT_MAX)
1907                 max_blocks = (u64)USHRT_MAX + 1;
1908         else
1909                 max_blocks = ns->ctrl->max_hw_sectors + 1;
1910
1911         blk_queue_max_write_zeroes_sectors(disk->queue,
1912                                            nvme_lba_to_sect(ns, max_blocks));
1913 }
1914
1915 static int nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1916                 struct nvme_id_ns *id, struct nvme_ns_ids *ids)
1917 {
1918         memset(ids, 0, sizeof(*ids));
1919
1920         if (ctrl->vs >= NVME_VS(1, 1, 0))
1921                 memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
1922         if (ctrl->vs >= NVME_VS(1, 2, 0))
1923                 memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
1924         if (ctrl->vs >= NVME_VS(1, 3, 0) || nvme_multi_css(ctrl))
1925                 return nvme_identify_ns_descs(ctrl, nsid, ids);
1926         return 0;
1927 }
1928
1929 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1930 {
1931         return !uuid_is_null(&ids->uuid) ||
1932                 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1933                 memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1934 }
1935
1936 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1937 {
1938         return uuid_equal(&a->uuid, &b->uuid) &&
1939                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1940                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1941                 a->csi == b->csi;
1942 }
1943
1944 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1945                                  u32 *phys_bs, u32 *io_opt)
1946 {
1947         struct streams_directive_params s;
1948         int ret;
1949
1950         if (!ctrl->nr_streams)
1951                 return 0;
1952
1953         ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
1954         if (ret)
1955                 return ret;
1956
1957         ns->sws = le32_to_cpu(s.sws);
1958         ns->sgs = le16_to_cpu(s.sgs);
1959
1960         if (ns->sws) {
1961                 *phys_bs = ns->sws * (1 << ns->lba_shift);
1962                 if (ns->sgs)
1963                         *io_opt = *phys_bs * ns->sgs;
1964         }
1965
1966         return 0;
1967 }
1968
1969 static void nvme_update_disk_info(struct gendisk *disk,
1970                 struct nvme_ns *ns, struct nvme_id_ns *id)
1971 {
1972         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1973         unsigned short bs = 1 << ns->lba_shift;
1974         u32 atomic_bs, phys_bs, io_opt = 0;
1975
1976         if (ns->lba_shift > PAGE_SHIFT) {
1977                 /* unsupported block size, set capacity to 0 later */
1978                 bs = (1 << 9);
1979         }
1980         blk_mq_freeze_queue(disk->queue);
1981         blk_integrity_unregister(disk);
1982
1983         atomic_bs = phys_bs = bs;
1984         nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
1985         if (id->nabo == 0) {
1986                 /*
1987                  * Bit 1 indicates whether NAWUPF is defined for this namespace
1988                  * and whether it should be used instead of AWUPF. If NAWUPF ==
1989                  * 0 then AWUPF must be used instead.
1990                  */
1991                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1992                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1993                 else
1994                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1995         }
1996
1997         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1998                 /* NPWG = Namespace Preferred Write Granularity */
1999                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
2000                 /* NOWS = Namespace Optimal Write Size */
2001                 io_opt = bs * (1 + le16_to_cpu(id->nows));
2002         }
2003
2004         blk_queue_logical_block_size(disk->queue, bs);
2005         /*
2006          * Linux filesystems assume writing a single physical block is
2007          * an atomic operation. Hence limit the physical block size to the
2008          * value of the Atomic Write Unit Power Fail parameter.
2009          */
2010         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
2011         blk_queue_io_min(disk->queue, phys_bs);
2012         blk_queue_io_opt(disk->queue, io_opt);
2013
2014         /*
2015          * The block layer can't support LBA sizes larger than the page size
2016          * yet, so catch this early and don't allow block I/O.
2017          */
2018         if (ns->lba_shift > PAGE_SHIFT)
2019                 capacity = 0;
2020
2021         /*
2022          * Register a metadata profile for PI, or the plain non-integrity NVMe
2023          * metadata masquerading as Type 0 if supported, otherwise reject block
2024          * I/O to namespaces with metadata except when the namespace supports
2025          * PI, as it can strip/insert in that case.
2026          */
2027         if (ns->ms) {
2028                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
2029                     (ns->features & NVME_NS_METADATA_SUPPORTED))
2030                         nvme_init_integrity(disk, ns->ms, ns->pi_type,
2031                                             ns->ctrl->max_integrity_segments);
2032                 else if (!nvme_ns_has_pi(ns))
2033                         capacity = 0;
2034         }
2035
2036         set_capacity_revalidate_and_notify(disk, capacity, false);
2037
2038         nvme_config_discard(disk, ns);
2039         nvme_config_write_zeroes(disk, ns);
2040
2041         if (id->nsattr & NVME_NS_ATTR_RO)
2042                 set_disk_ro(disk, true);
2043         else
2044                 set_disk_ro(disk, false);
2045
2046         blk_mq_unfreeze_queue(disk->queue);
2047 }
2048
2049 static inline bool nvme_first_scan(struct gendisk *disk)
2050 {
2051         /* nvme_alloc_ns() scans the disk prior to adding it */
2052         return !(disk->flags & GENHD_FL_UP);
2053 }
2054
2055 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
2056 {
2057         struct nvme_ctrl *ctrl = ns->ctrl;
2058         u32 iob;
2059
2060         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2061             is_power_of_2(ctrl->max_hw_sectors))
2062                 iob = ctrl->max_hw_sectors;
2063         else
2064                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
2065
2066         if (!iob)
2067                 return;
2068
2069         if (!is_power_of_2(iob)) {
2070                 if (nvme_first_scan(ns->disk))
2071                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2072                                 ns->disk->disk_name, iob);
2073                 return;
2074         }
2075
2076         if (blk_queue_is_zoned(ns->disk->queue)) {
2077                 if (nvme_first_scan(ns->disk))
2078                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
2079                                 ns->disk->disk_name);
2080                 return;
2081         }
2082
2083         blk_queue_chunk_sectors(ns->queue, iob);
2084 }
2085
2086 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
2087 {
2088         unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2089         struct nvme_ctrl *ctrl = ns->ctrl;
2090         int ret;
2091
2092         /*
2093          * If identify namespace failed, use default 512 byte block size so
2094          * block layer can use before failing read/write for 0 capacity.
2095          */
2096         ns->lba_shift = id->lbaf[lbaf].ds;
2097         if (ns->lba_shift == 0)
2098                 ns->lba_shift = 9;
2099
2100         switch (ns->head->ids.csi) {
2101         case NVME_CSI_NVM:
2102                 break;
2103         case NVME_CSI_ZNS:
2104                 ret = nvme_update_zone_info(ns, lbaf);
2105                 if (ret) {
2106                         dev_warn(ctrl->device,
2107                                 "failed to add zoned namespace:%u ret:%d\n",
2108                                 ns->head->ns_id, ret);
2109                         return ret;
2110                 }
2111                 break;
2112         default:
2113                 dev_warn(ctrl->device, "unknown csi:%u ns:%u\n",
2114                         ns->head->ids.csi, ns->head->ns_id);
2115                 return -ENODEV;
2116         }
2117
2118         ns->features = 0;
2119         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
2120         /* the PI implementation requires metadata equal t10 pi tuple size */
2121         if (ns->ms == sizeof(struct t10_pi_tuple))
2122                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
2123         else
2124                 ns->pi_type = 0;
2125
2126         if (ns->ms) {
2127                 /*
2128                  * For PCIe only the separate metadata pointer is supported,
2129                  * as the block layer supplies metadata in a separate bio_vec
2130                  * chain. For Fabrics, only metadata as part of extended data
2131                  * LBA is supported on the wire per the Fabrics specification,
2132                  * but the HBA/HCA will do the remapping from the separate
2133                  * metadata buffers for us.
2134                  */
2135                 if (id->flbas & NVME_NS_FLBAS_META_EXT) {
2136                         ns->features |= NVME_NS_EXT_LBAS;
2137                         if ((ctrl->ops->flags & NVME_F_FABRICS) &&
2138                             (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) &&
2139                             ctrl->max_integrity_segments)
2140                                 ns->features |= NVME_NS_METADATA_SUPPORTED;
2141                 } else {
2142                         if (WARN_ON_ONCE(ctrl->ops->flags & NVME_F_FABRICS))
2143                                 return -EINVAL;
2144                         if (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)
2145                                 ns->features |= NVME_NS_METADATA_SUPPORTED;
2146                 }
2147         }
2148
2149         nvme_set_chunk_sectors(ns, id);
2150         nvme_update_disk_info(ns->disk, ns, id);
2151 #ifdef CONFIG_NVME_MULTIPATH
2152         if (ns->head->disk) {
2153                 nvme_update_disk_info(ns->head->disk, ns, id);
2154                 blk_stack_limits(&ns->head->disk->queue->limits,
2155                                  &ns->queue->limits, 0);
2156                 blk_queue_update_readahead(ns->head->disk->queue);
2157                 nvme_update_bdev_size(ns->head->disk);
2158         }
2159 #endif
2160         return 0;
2161 }
2162
2163 static int nvme_validate_ns(struct nvme_ns *ns)
2164 {
2165         struct nvme_ctrl *ctrl = ns->ctrl;
2166         struct nvme_id_ns *id;
2167         struct nvme_ns_ids ids;
2168         int ret = 0;
2169
2170         if (test_bit(NVME_NS_DEAD, &ns->flags)) {
2171                 set_capacity(ns->disk, 0);
2172                 return -ENODEV;
2173         }
2174
2175         ret = nvme_identify_ns(ctrl, ns->head->ns_id, &id);
2176         if (ret)
2177                 goto out;
2178
2179         ret = nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
2180         if (ret)
2181                 goto free_id;
2182
2183         if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
2184                 dev_err(ctrl->device,
2185                         "identifiers changed for nsid %d\n", ns->head->ns_id);
2186                 ret = -ENODEV;
2187                 goto free_id;
2188         }
2189
2190         ret = nvme_update_ns_info(ns, id);
2191 free_id:
2192         kfree(id);
2193 out:
2194         /*
2195          * Only fail the function if we got a fatal error back from the
2196          * device, otherwise ignore the error and just move on.
2197          */
2198         if (ret == -ENOMEM || (ret > 0 && !(ret & NVME_SC_DNR)))
2199                 ret = 0;
2200         else if (ret > 0)
2201                 ret = blk_status_to_errno(nvme_error_status(ret));
2202         return ret;
2203 }
2204
2205 static char nvme_pr_type(enum pr_type type)
2206 {
2207         switch (type) {
2208         case PR_WRITE_EXCLUSIVE:
2209                 return 1;
2210         case PR_EXCLUSIVE_ACCESS:
2211                 return 2;
2212         case PR_WRITE_EXCLUSIVE_REG_ONLY:
2213                 return 3;
2214         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2215                 return 4;
2216         case PR_WRITE_EXCLUSIVE_ALL_REGS:
2217                 return 5;
2218         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2219                 return 6;
2220         default:
2221                 return 0;
2222         }
2223 };
2224
2225 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2226                                 u64 key, u64 sa_key, u8 op)
2227 {
2228         struct nvme_ns_head *head = NULL;
2229         struct nvme_ns *ns;
2230         struct nvme_command c;
2231         int srcu_idx, ret;
2232         u8 data[16] = { 0, };
2233
2234         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
2235         if (unlikely(!ns))
2236                 return -EWOULDBLOCK;
2237
2238         put_unaligned_le64(key, &data[0]);
2239         put_unaligned_le64(sa_key, &data[8]);
2240
2241         memset(&c, 0, sizeof(c));
2242         c.common.opcode = op;
2243         c.common.nsid = cpu_to_le32(ns->head->ns_id);
2244         c.common.cdw10 = cpu_to_le32(cdw10);
2245
2246         ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
2247         nvme_put_ns_from_disk(head, srcu_idx);
2248         return ret;
2249 }
2250
2251 static int nvme_pr_register(struct block_device *bdev, u64 old,
2252                 u64 new, unsigned flags)
2253 {
2254         u32 cdw10;
2255
2256         if (flags & ~PR_FL_IGNORE_KEY)
2257                 return -EOPNOTSUPP;
2258
2259         cdw10 = old ? 2 : 0;
2260         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2261         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2262         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2263 }
2264
2265 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2266                 enum pr_type type, unsigned flags)
2267 {
2268         u32 cdw10;
2269
2270         if (flags & ~PR_FL_IGNORE_KEY)
2271                 return -EOPNOTSUPP;
2272
2273         cdw10 = nvme_pr_type(type) << 8;
2274         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2275         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2276 }
2277
2278 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2279                 enum pr_type type, bool abort)
2280 {
2281         u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2282         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2283 }
2284
2285 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2286 {
2287         u32 cdw10 = 1 | (key ? 1 << 3 : 0);
2288         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2289 }
2290
2291 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2292 {
2293         u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
2294         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2295 }
2296
2297 static const struct pr_ops nvme_pr_ops = {
2298         .pr_register    = nvme_pr_register,
2299         .pr_reserve     = nvme_pr_reserve,
2300         .pr_release     = nvme_pr_release,
2301         .pr_preempt     = nvme_pr_preempt,
2302         .pr_clear       = nvme_pr_clear,
2303 };
2304
2305 #ifdef CONFIG_BLK_SED_OPAL
2306 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2307                 bool send)
2308 {
2309         struct nvme_ctrl *ctrl = data;
2310         struct nvme_command cmd;
2311
2312         memset(&cmd, 0, sizeof(cmd));
2313         if (send)
2314                 cmd.common.opcode = nvme_admin_security_send;
2315         else
2316                 cmd.common.opcode = nvme_admin_security_recv;
2317         cmd.common.nsid = 0;
2318         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2319         cmd.common.cdw11 = cpu_to_le32(len);
2320
2321         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2322                                       ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2323 }
2324 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2325 #endif /* CONFIG_BLK_SED_OPAL */
2326
2327 static const struct block_device_operations nvme_fops = {
2328         .owner          = THIS_MODULE,
2329         .ioctl          = nvme_ioctl,
2330         .compat_ioctl   = nvme_compat_ioctl,
2331         .open           = nvme_open,
2332         .release        = nvme_release,
2333         .getgeo         = nvme_getgeo,
2334         .report_zones   = nvme_report_zones,
2335         .pr_ops         = &nvme_pr_ops,
2336 };
2337
2338 #ifdef CONFIG_NVME_MULTIPATH
2339 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2340 {
2341         struct nvme_ns_head *head = bdev->bd_disk->private_data;
2342
2343         if (!kref_get_unless_zero(&head->ref))
2344                 return -ENXIO;
2345         return 0;
2346 }
2347
2348 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2349 {
2350         nvme_put_ns_head(disk->private_data);
2351 }
2352
2353 const struct block_device_operations nvme_ns_head_ops = {
2354         .owner          = THIS_MODULE,
2355         .submit_bio     = nvme_ns_head_submit_bio,
2356         .open           = nvme_ns_head_open,
2357         .release        = nvme_ns_head_release,
2358         .ioctl          = nvme_ioctl,
2359         .compat_ioctl   = nvme_compat_ioctl,
2360         .getgeo         = nvme_getgeo,
2361         .report_zones   = nvme_report_zones,
2362         .pr_ops         = &nvme_pr_ops,
2363 };
2364 #endif /* CONFIG_NVME_MULTIPATH */
2365
2366 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2367 {
2368         unsigned long timeout =
2369                 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2370         u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2371         int ret;
2372
2373         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2374                 if (csts == ~0)
2375                         return -ENODEV;
2376                 if ((csts & NVME_CSTS_RDY) == bit)
2377                         break;
2378
2379                 usleep_range(1000, 2000);
2380                 if (fatal_signal_pending(current))
2381                         return -EINTR;
2382                 if (time_after(jiffies, timeout)) {
2383                         dev_err(ctrl->device,
2384                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2385                                 enabled ? "initialisation" : "reset", csts);
2386                         return -ENODEV;
2387                 }
2388         }
2389
2390         return ret;
2391 }
2392
2393 /*
2394  * If the device has been passed off to us in an enabled state, just clear
2395  * the enabled bit.  The spec says we should set the 'shutdown notification
2396  * bits', but doing so may cause the device to complete commands to the
2397  * admin queue ... and we don't know what memory that might be pointing at!
2398  */
2399 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2400 {
2401         int ret;
2402
2403         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2404         ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2405
2406         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2407         if (ret)
2408                 return ret;
2409
2410         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2411                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2412
2413         return nvme_wait_ready(ctrl, ctrl->cap, false);
2414 }
2415 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2416
2417 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2418 {
2419         unsigned dev_page_min;
2420         int ret;
2421
2422         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2423         if (ret) {
2424                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2425                 return ret;
2426         }
2427         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2428
2429         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2430                 dev_err(ctrl->device,
2431                         "Minimum device page size %u too large for host (%u)\n",
2432                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2433                 return -ENODEV;
2434         }
2435
2436         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2437                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2438         else
2439                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2440         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2441         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2442         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2443         ctrl->ctrl_config |= NVME_CC_ENABLE;
2444
2445         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2446         if (ret)
2447                 return ret;
2448         return nvme_wait_ready(ctrl, ctrl->cap, true);
2449 }
2450 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2451
2452 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2453 {
2454         unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2455         u32 csts;
2456         int ret;
2457
2458         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2459         ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2460
2461         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2462         if (ret)
2463                 return ret;
2464
2465         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2466                 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2467                         break;
2468
2469                 msleep(100);
2470                 if (fatal_signal_pending(current))
2471                         return -EINTR;
2472                 if (time_after(jiffies, timeout)) {
2473                         dev_err(ctrl->device,
2474                                 "Device shutdown incomplete; abort shutdown\n");
2475                         return -ENODEV;
2476                 }
2477         }
2478
2479         return ret;
2480 }
2481 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2482
2483 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
2484                 struct request_queue *q)
2485 {
2486         bool vwc = false;
2487
2488         if (ctrl->max_hw_sectors) {
2489                 u32 max_segments =
2490                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
2491
2492                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
2493                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
2494                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
2495         }
2496         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
2497         blk_queue_dma_alignment(q, 7);
2498         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2499                 vwc = true;
2500         blk_queue_write_cache(q, vwc, vwc);
2501 }
2502
2503 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2504 {
2505         __le64 ts;
2506         int ret;
2507
2508         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2509                 return 0;
2510
2511         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2512         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2513                         NULL);
2514         if (ret)
2515                 dev_warn_once(ctrl->device,
2516                         "could not set timestamp (%d)\n", ret);
2517         return ret;
2518 }
2519
2520 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2521 {
2522         struct nvme_feat_host_behavior *host;
2523         int ret;
2524
2525         /* Don't bother enabling the feature if retry delay is not reported */
2526         if (!ctrl->crdt[0])
2527                 return 0;
2528
2529         host = kzalloc(sizeof(*host), GFP_KERNEL);
2530         if (!host)
2531                 return 0;
2532
2533         host->acre = NVME_ENABLE_ACRE;
2534         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2535                                 host, sizeof(*host), NULL);
2536         kfree(host);
2537         return ret;
2538 }
2539
2540 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2541 {
2542         /*
2543          * APST (Autonomous Power State Transition) lets us program a
2544          * table of power state transitions that the controller will
2545          * perform automatically.  We configure it with a simple
2546          * heuristic: we are willing to spend at most 2% of the time
2547          * transitioning between power states.  Therefore, when running
2548          * in any given state, we will enter the next lower-power
2549          * non-operational state after waiting 50 * (enlat + exlat)
2550          * microseconds, as long as that state's exit latency is under
2551          * the requested maximum latency.
2552          *
2553          * We will not autonomously enter any non-operational state for
2554          * which the total latency exceeds ps_max_latency_us.  Users
2555          * can set ps_max_latency_us to zero to turn off APST.
2556          */
2557
2558         unsigned apste;
2559         struct nvme_feat_auto_pst *table;
2560         u64 max_lat_us = 0;
2561         int max_ps = -1;
2562         int ret;
2563
2564         /*
2565          * If APST isn't supported or if we haven't been initialized yet,
2566          * then don't do anything.
2567          */
2568         if (!ctrl->apsta)
2569                 return 0;
2570
2571         if (ctrl->npss > 31) {
2572                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2573                 return 0;
2574         }
2575
2576         table = kzalloc(sizeof(*table), GFP_KERNEL);
2577         if (!table)
2578                 return 0;
2579
2580         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2581                 /* Turn off APST. */
2582                 apste = 0;
2583                 dev_dbg(ctrl->device, "APST disabled\n");
2584         } else {
2585                 __le64 target = cpu_to_le64(0);
2586                 int state;
2587
2588                 /*
2589                  * Walk through all states from lowest- to highest-power.
2590                  * According to the spec, lower-numbered states use more
2591                  * power.  NPSS, despite the name, is the index of the
2592                  * lowest-power state, not the number of states.
2593                  */
2594                 for (state = (int)ctrl->npss; state >= 0; state--) {
2595                         u64 total_latency_us, exit_latency_us, transition_ms;
2596
2597                         if (target)
2598                                 table->entries[state] = target;
2599
2600                         /*
2601                          * Don't allow transitions to the deepest state
2602                          * if it's quirked off.
2603                          */
2604                         if (state == ctrl->npss &&
2605                             (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2606                                 continue;
2607
2608                         /*
2609                          * Is this state a useful non-operational state for
2610                          * higher-power states to autonomously transition to?
2611                          */
2612                         if (!(ctrl->psd[state].flags &
2613                               NVME_PS_FLAGS_NON_OP_STATE))
2614                                 continue;
2615
2616                         exit_latency_us =
2617                                 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2618                         if (exit_latency_us > ctrl->ps_max_latency_us)
2619                                 continue;
2620
2621                         total_latency_us =
2622                                 exit_latency_us +
2623                                 le32_to_cpu(ctrl->psd[state].entry_lat);
2624
2625                         /*
2626                          * This state is good.  Use it as the APST idle
2627                          * target for higher power states.
2628                          */
2629                         transition_ms = total_latency_us + 19;
2630                         do_div(transition_ms, 20);
2631                         if (transition_ms > (1 << 24) - 1)
2632                                 transition_ms = (1 << 24) - 1;
2633
2634                         target = cpu_to_le64((state << 3) |
2635                                              (transition_ms << 8));
2636
2637                         if (max_ps == -1)
2638                                 max_ps = state;
2639
2640                         if (total_latency_us > max_lat_us)
2641                                 max_lat_us = total_latency_us;
2642                 }
2643
2644                 apste = 1;
2645
2646                 if (max_ps == -1) {
2647                         dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2648                 } else {
2649                         dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2650                                 max_ps, max_lat_us, (int)sizeof(*table), table);
2651                 }
2652         }
2653
2654         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2655                                 table, sizeof(*table), NULL);
2656         if (ret)
2657                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2658
2659         kfree(table);
2660         return ret;
2661 }
2662
2663 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2664 {
2665         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2666         u64 latency;
2667
2668         switch (val) {
2669         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2670         case PM_QOS_LATENCY_ANY:
2671                 latency = U64_MAX;
2672                 break;
2673
2674         default:
2675                 latency = val;
2676         }
2677
2678         if (ctrl->ps_max_latency_us != latency) {
2679                 ctrl->ps_max_latency_us = latency;
2680                 nvme_configure_apst(ctrl);
2681         }
2682 }
2683
2684 struct nvme_core_quirk_entry {
2685         /*
2686          * NVMe model and firmware strings are padded with spaces.  For
2687          * simplicity, strings in the quirk table are padded with NULLs
2688          * instead.
2689          */
2690         u16 vid;
2691         const char *mn;
2692         const char *fr;
2693         unsigned long quirks;
2694 };
2695
2696 static const struct nvme_core_quirk_entry core_quirks[] = {
2697         {
2698                 /*
2699                  * This Toshiba device seems to die using any APST states.  See:
2700                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2701                  */
2702                 .vid = 0x1179,
2703                 .mn = "THNSF5256GPUK TOSHIBA",
2704                 .quirks = NVME_QUIRK_NO_APST,
2705         },
2706         {
2707                 /*
2708                  * This LiteON CL1-3D*-Q11 firmware version has a race
2709                  * condition associated with actions related to suspend to idle
2710                  * LiteON has resolved the problem in future firmware
2711                  */
2712                 .vid = 0x14a4,
2713                 .fr = "22301111",
2714                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2715         }
2716 };
2717
2718 /* match is null-terminated but idstr is space-padded. */
2719 static bool string_matches(const char *idstr, const char *match, size_t len)
2720 {
2721         size_t matchlen;
2722
2723         if (!match)
2724                 return true;
2725
2726         matchlen = strlen(match);
2727         WARN_ON_ONCE(matchlen > len);
2728
2729         if (memcmp(idstr, match, matchlen))
2730                 return false;
2731
2732         for (; matchlen < len; matchlen++)
2733                 if (idstr[matchlen] != ' ')
2734                         return false;
2735
2736         return true;
2737 }
2738
2739 static bool quirk_matches(const struct nvme_id_ctrl *id,
2740                           const struct nvme_core_quirk_entry *q)
2741 {
2742         return q->vid == le16_to_cpu(id->vid) &&
2743                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2744                 string_matches(id->fr, q->fr, sizeof(id->fr));
2745 }
2746
2747 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2748                 struct nvme_id_ctrl *id)
2749 {
2750         size_t nqnlen;
2751         int off;
2752
2753         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2754                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2755                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2756                         strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2757                         return;
2758                 }
2759
2760                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2761                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2762         }
2763
2764         /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2765         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2766                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2767                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2768         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2769         off += sizeof(id->sn);
2770         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2771         off += sizeof(id->mn);
2772         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2773 }
2774
2775 static void nvme_release_subsystem(struct device *dev)
2776 {
2777         struct nvme_subsystem *subsys =
2778                 container_of(dev, struct nvme_subsystem, dev);
2779
2780         if (subsys->instance >= 0)
2781                 ida_simple_remove(&nvme_instance_ida, subsys->instance);
2782         kfree(subsys);
2783 }
2784
2785 static void nvme_destroy_subsystem(struct kref *ref)
2786 {
2787         struct nvme_subsystem *subsys =
2788                         container_of(ref, struct nvme_subsystem, ref);
2789
2790         mutex_lock(&nvme_subsystems_lock);
2791         list_del(&subsys->entry);
2792         mutex_unlock(&nvme_subsystems_lock);
2793
2794         ida_destroy(&subsys->ns_ida);
2795         device_del(&subsys->dev);
2796         put_device(&subsys->dev);
2797 }
2798
2799 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2800 {
2801         kref_put(&subsys->ref, nvme_destroy_subsystem);
2802 }
2803
2804 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2805 {
2806         struct nvme_subsystem *subsys;
2807
2808         lockdep_assert_held(&nvme_subsystems_lock);
2809
2810         /*
2811          * Fail matches for discovery subsystems. This results
2812          * in each discovery controller bound to a unique subsystem.
2813          * This avoids issues with validating controller values
2814          * that can only be true when there is a single unique subsystem.
2815          * There may be multiple and completely independent entities
2816          * that provide discovery controllers.
2817          */
2818         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2819                 return NULL;
2820
2821         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2822                 if (strcmp(subsys->subnqn, subsysnqn))
2823                         continue;
2824                 if (!kref_get_unless_zero(&subsys->ref))
2825                         continue;
2826                 return subsys;
2827         }
2828
2829         return NULL;
2830 }
2831
2832 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2833         struct device_attribute subsys_attr_##_name = \
2834                 __ATTR(_name, _mode, _show, NULL)
2835
2836 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2837                                     struct device_attribute *attr,
2838                                     char *buf)
2839 {
2840         struct nvme_subsystem *subsys =
2841                 container_of(dev, struct nvme_subsystem, dev);
2842
2843         return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2844 }
2845 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2846
2847 #define nvme_subsys_show_str_function(field)                            \
2848 static ssize_t subsys_##field##_show(struct device *dev,                \
2849                             struct device_attribute *attr, char *buf)   \
2850 {                                                                       \
2851         struct nvme_subsystem *subsys =                                 \
2852                 container_of(dev, struct nvme_subsystem, dev);          \
2853         return sprintf(buf, "%.*s\n",                                   \
2854                        (int)sizeof(subsys->field), subsys->field);      \
2855 }                                                                       \
2856 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2857
2858 nvme_subsys_show_str_function(model);
2859 nvme_subsys_show_str_function(serial);
2860 nvme_subsys_show_str_function(firmware_rev);
2861
2862 static struct attribute *nvme_subsys_attrs[] = {
2863         &subsys_attr_model.attr,
2864         &subsys_attr_serial.attr,
2865         &subsys_attr_firmware_rev.attr,
2866         &subsys_attr_subsysnqn.attr,
2867 #ifdef CONFIG_NVME_MULTIPATH
2868         &subsys_attr_iopolicy.attr,
2869 #endif
2870         NULL,
2871 };
2872
2873 static struct attribute_group nvme_subsys_attrs_group = {
2874         .attrs = nvme_subsys_attrs,
2875 };
2876
2877 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2878         &nvme_subsys_attrs_group,
2879         NULL,
2880 };
2881
2882 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2883                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2884 {
2885         struct nvme_ctrl *tmp;
2886
2887         lockdep_assert_held(&nvme_subsystems_lock);
2888
2889         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2890                 if (nvme_state_terminal(tmp))
2891                         continue;
2892
2893                 if (tmp->cntlid == ctrl->cntlid) {
2894                         dev_err(ctrl->device,
2895                                 "Duplicate cntlid %u with %s, rejecting\n",
2896                                 ctrl->cntlid, dev_name(tmp->device));
2897                         return false;
2898                 }
2899
2900                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2901                     (ctrl->opts && ctrl->opts->discovery_nqn))
2902                         continue;
2903
2904                 dev_err(ctrl->device,
2905                         "Subsystem does not support multiple controllers\n");
2906                 return false;
2907         }
2908
2909         return true;
2910 }
2911
2912 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2913 {
2914         struct nvme_subsystem *subsys, *found;
2915         int ret;
2916
2917         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2918         if (!subsys)
2919                 return -ENOMEM;
2920
2921         subsys->instance = -1;
2922         mutex_init(&subsys->lock);
2923         kref_init(&subsys->ref);
2924         INIT_LIST_HEAD(&subsys->ctrls);
2925         INIT_LIST_HEAD(&subsys->nsheads);
2926         nvme_init_subnqn(subsys, ctrl, id);
2927         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2928         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2929         memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2930         subsys->vendor_id = le16_to_cpu(id->vid);
2931         subsys->cmic = id->cmic;
2932         subsys->awupf = le16_to_cpu(id->awupf);
2933 #ifdef CONFIG_NVME_MULTIPATH
2934         subsys->iopolicy = NVME_IOPOLICY_NUMA;
2935 #endif
2936
2937         subsys->dev.class = nvme_subsys_class;
2938         subsys->dev.release = nvme_release_subsystem;
2939         subsys->dev.groups = nvme_subsys_attrs_groups;
2940         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2941         device_initialize(&subsys->dev);
2942
2943         mutex_lock(&nvme_subsystems_lock);
2944         found = __nvme_find_get_subsystem(subsys->subnqn);
2945         if (found) {
2946                 put_device(&subsys->dev);
2947                 subsys = found;
2948
2949                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2950                         ret = -EINVAL;
2951                         goto out_put_subsystem;
2952                 }
2953         } else {
2954                 ret = device_add(&subsys->dev);
2955                 if (ret) {
2956                         dev_err(ctrl->device,
2957                                 "failed to register subsystem device.\n");
2958                         put_device(&subsys->dev);
2959                         goto out_unlock;
2960                 }
2961                 ida_init(&subsys->ns_ida);
2962                 list_add_tail(&subsys->entry, &nvme_subsystems);
2963         }
2964
2965         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2966                                 dev_name(ctrl->device));
2967         if (ret) {
2968                 dev_err(ctrl->device,
2969                         "failed to create sysfs link from subsystem.\n");
2970                 goto out_put_subsystem;
2971         }
2972
2973         if (!found)
2974                 subsys->instance = ctrl->instance;
2975         ctrl->subsys = subsys;
2976         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2977         mutex_unlock(&nvme_subsystems_lock);
2978         return 0;
2979
2980 out_put_subsystem:
2981         nvme_put_subsystem(subsys);
2982 out_unlock:
2983         mutex_unlock(&nvme_subsystems_lock);
2984         return ret;
2985 }
2986
2987 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2988                 void *log, size_t size, u64 offset)
2989 {
2990         struct nvme_command c = { };
2991         u32 dwlen = nvme_bytes_to_numd(size);
2992
2993         c.get_log_page.opcode = nvme_admin_get_log_page;
2994         c.get_log_page.nsid = cpu_to_le32(nsid);
2995         c.get_log_page.lid = log_page;
2996         c.get_log_page.lsp = lsp;
2997         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2998         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2999         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
3000         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3001         c.get_log_page.csi = csi;
3002
3003         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3004 }
3005
3006 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3007                                 struct nvme_effects_log **log)
3008 {
3009         struct nvme_cel *cel = xa_load(&ctrl->cels, csi);
3010         int ret;
3011
3012         if (cel)
3013                 goto out;
3014
3015         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3016         if (!cel)
3017                 return -ENOMEM;
3018
3019         ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0, csi,
3020                         &cel->log, sizeof(cel->log), 0);
3021         if (ret) {
3022                 kfree(cel);
3023                 return ret;
3024         }
3025
3026         cel->csi = csi;
3027         xa_store(&ctrl->cels, cel->csi, cel, GFP_KERNEL);
3028 out:
3029         *log = &cel->log;
3030         return 0;
3031 }
3032
3033 /*
3034  * Initialize the cached copies of the Identify data and various controller
3035  * register in our nvme_ctrl structure.  This should be called as soon as
3036  * the admin queue is fully up and running.
3037  */
3038 int nvme_init_identify(struct nvme_ctrl *ctrl)
3039 {
3040         struct nvme_id_ctrl *id;
3041         int ret, page_shift;
3042         u32 max_hw_sectors;
3043         bool prev_apst_enabled;
3044
3045         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3046         if (ret) {
3047                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3048                 return ret;
3049         }
3050         page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
3051         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3052
3053         if (ctrl->vs >= NVME_VS(1, 1, 0))
3054                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3055
3056         ret = nvme_identify_ctrl(ctrl, &id);
3057         if (ret) {
3058                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3059                 return -EIO;
3060         }
3061
3062         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3063                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3064                 if (ret < 0)
3065                         goto out_free;
3066         }
3067
3068         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3069                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3070
3071         if (!ctrl->identified) {
3072                 int i;
3073
3074                 ret = nvme_init_subsystem(ctrl, id);
3075                 if (ret)
3076                         goto out_free;
3077
3078                 /*
3079                  * Check for quirks.  Quirk can depend on firmware version,
3080                  * so, in principle, the set of quirks present can change
3081                  * across a reset.  As a possible future enhancement, we
3082                  * could re-scan for quirks every time we reinitialize
3083                  * the device, but we'd have to make sure that the driver
3084                  * behaves intelligently if the quirks change.
3085                  */
3086                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3087                         if (quirk_matches(id, &core_quirks[i]))
3088                                 ctrl->quirks |= core_quirks[i].quirks;
3089                 }
3090         }
3091
3092         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3093                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3094                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3095         }
3096
3097         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3098         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3099         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3100
3101         ctrl->oacs = le16_to_cpu(id->oacs);
3102         ctrl->oncs = le16_to_cpu(id->oncs);
3103         ctrl->mtfa = le16_to_cpu(id->mtfa);
3104         ctrl->oaes = le32_to_cpu(id->oaes);
3105         ctrl->wctemp = le16_to_cpu(id->wctemp);
3106         ctrl->cctemp = le16_to_cpu(id->cctemp);
3107
3108         atomic_set(&ctrl->abort_limit, id->acl + 1);
3109         ctrl->vwc = id->vwc;
3110         if (id->mdts)
3111                 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
3112         else
3113                 max_hw_sectors = UINT_MAX;
3114         ctrl->max_hw_sectors =
3115                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3116
3117         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3118         ctrl->sgls = le32_to_cpu(id->sgls);
3119         ctrl->kas = le16_to_cpu(id->kas);
3120         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3121         ctrl->ctratt = le32_to_cpu(id->ctratt);
3122
3123         if (id->rtd3e) {
3124                 /* us -> s */
3125                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3126
3127                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3128                                                  shutdown_timeout, 60);
3129
3130                 if (ctrl->shutdown_timeout != shutdown_timeout)
3131                         dev_info(ctrl->device,
3132                                  "Shutdown timeout set to %u seconds\n",
3133                                  ctrl->shutdown_timeout);
3134         } else
3135                 ctrl->shutdown_timeout = shutdown_timeout;
3136
3137         ctrl->npss = id->npss;
3138         ctrl->apsta = id->apsta;
3139         prev_apst_enabled = ctrl->apst_enabled;
3140         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3141                 if (force_apst && id->apsta) {
3142                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3143                         ctrl->apst_enabled = true;
3144                 } else {
3145                         ctrl->apst_enabled = false;
3146                 }
3147         } else {
3148                 ctrl->apst_enabled = id->apsta;
3149         }
3150         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3151
3152         if (ctrl->ops->flags & NVME_F_FABRICS) {
3153                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3154                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3155                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3156                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3157
3158                 /*
3159                  * In fabrics we need to verify the cntlid matches the
3160                  * admin connect
3161                  */
3162                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3163                         dev_err(ctrl->device,
3164                                 "Mismatching cntlid: Connect %u vs Identify "
3165                                 "%u, rejecting\n",
3166                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3167                         ret = -EINVAL;
3168                         goto out_free;
3169                 }
3170
3171                 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
3172                         dev_err(ctrl->device,
3173                                 "keep-alive support is mandatory for fabrics\n");
3174                         ret = -EINVAL;
3175                         goto out_free;
3176                 }
3177         } else {
3178                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3179                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3180                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3181                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3182         }
3183
3184         ret = nvme_mpath_init(ctrl, id);
3185         kfree(id);
3186
3187         if (ret < 0)
3188                 return ret;
3189
3190         if (ctrl->apst_enabled && !prev_apst_enabled)
3191                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3192         else if (!ctrl->apst_enabled && prev_apst_enabled)
3193                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3194
3195         ret = nvme_configure_apst(ctrl);
3196         if (ret < 0)
3197                 return ret;
3198
3199         ret = nvme_configure_timestamp(ctrl);
3200         if (ret < 0)
3201                 return ret;
3202
3203         ret = nvme_configure_directives(ctrl);
3204         if (ret < 0)
3205                 return ret;
3206
3207         ret = nvme_configure_acre(ctrl);
3208         if (ret < 0)
3209                 return ret;
3210
3211         if (!ctrl->identified)
3212                 nvme_hwmon_init(ctrl);
3213
3214         ctrl->identified = true;
3215
3216         return 0;
3217
3218 out_free:
3219         kfree(id);
3220         return ret;
3221 }
3222 EXPORT_SYMBOL_GPL(nvme_init_identify);
3223
3224 static int nvme_dev_open(struct inode *inode, struct file *file)
3225 {
3226         struct nvme_ctrl *ctrl =
3227                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3228
3229         switch (ctrl->state) {
3230         case NVME_CTRL_LIVE:
3231                 break;
3232         default:
3233                 return -EWOULDBLOCK;
3234         }
3235
3236         nvme_get_ctrl(ctrl);
3237         if (!try_module_get(ctrl->ops->module))
3238                 return -EINVAL;
3239
3240         file->private_data = ctrl;
3241         return 0;
3242 }
3243
3244 static int nvme_dev_release(struct inode *inode, struct file *file)
3245 {
3246         struct nvme_ctrl *ctrl =
3247                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3248
3249         module_put(ctrl->ops->module);
3250         nvme_put_ctrl(ctrl);
3251         return 0;
3252 }
3253
3254 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
3255 {
3256         struct nvme_ns *ns;
3257         int ret;
3258
3259         down_read(&ctrl->namespaces_rwsem);
3260         if (list_empty(&ctrl->namespaces)) {
3261                 ret = -ENOTTY;
3262                 goto out_unlock;
3263         }
3264
3265         ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
3266         if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
3267                 dev_warn(ctrl->device,
3268                         "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
3269                 ret = -EINVAL;
3270                 goto out_unlock;
3271         }
3272
3273         dev_warn(ctrl->device,
3274                 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
3275         kref_get(&ns->kref);
3276         up_read(&ctrl->namespaces_rwsem);
3277
3278         ret = nvme_user_cmd(ctrl, ns, argp);
3279         nvme_put_ns(ns);
3280         return ret;
3281
3282 out_unlock:
3283         up_read(&ctrl->namespaces_rwsem);
3284         return ret;
3285 }
3286
3287 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
3288                 unsigned long arg)
3289 {
3290         struct nvme_ctrl *ctrl = file->private_data;
3291         void __user *argp = (void __user *)arg;
3292
3293         switch (cmd) {
3294         case NVME_IOCTL_ADMIN_CMD:
3295                 return nvme_user_cmd(ctrl, NULL, argp);
3296         case NVME_IOCTL_ADMIN64_CMD:
3297                 return nvme_user_cmd64(ctrl, NULL, argp);
3298         case NVME_IOCTL_IO_CMD:
3299                 return nvme_dev_user_cmd(ctrl, argp);
3300         case NVME_IOCTL_RESET:
3301                 dev_warn(ctrl->device, "resetting controller\n");
3302                 return nvme_reset_ctrl_sync(ctrl);
3303         case NVME_IOCTL_SUBSYS_RESET:
3304                 return nvme_reset_subsystem(ctrl);
3305         case NVME_IOCTL_RESCAN:
3306                 nvme_queue_scan(ctrl);
3307                 return 0;
3308         default:
3309                 return -ENOTTY;
3310         }
3311 }
3312
3313 static const struct file_operations nvme_dev_fops = {
3314         .owner          = THIS_MODULE,
3315         .open           = nvme_dev_open,
3316         .release        = nvme_dev_release,
3317         .unlocked_ioctl = nvme_dev_ioctl,
3318         .compat_ioctl   = compat_ptr_ioctl,
3319 };
3320
3321 static ssize_t nvme_sysfs_reset(struct device *dev,
3322                                 struct device_attribute *attr, const char *buf,
3323                                 size_t count)
3324 {
3325         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3326         int ret;
3327
3328         ret = nvme_reset_ctrl_sync(ctrl);
3329         if (ret < 0)
3330                 return ret;
3331         return count;
3332 }
3333 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3334
3335 static ssize_t nvme_sysfs_rescan(struct device *dev,
3336                                 struct device_attribute *attr, const char *buf,
3337                                 size_t count)
3338 {
3339         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3340
3341         nvme_queue_scan(ctrl);
3342         return count;
3343 }
3344 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3345
3346 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3347 {
3348         struct gendisk *disk = dev_to_disk(dev);
3349
3350         if (disk->fops == &nvme_fops)
3351                 return nvme_get_ns_from_dev(dev)->head;
3352         else
3353                 return disk->private_data;
3354 }
3355
3356 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3357                 char *buf)
3358 {
3359         struct nvme_ns_head *head = dev_to_ns_head(dev);
3360         struct nvme_ns_ids *ids = &head->ids;
3361         struct nvme_subsystem *subsys = head->subsys;
3362         int serial_len = sizeof(subsys->serial);
3363         int model_len = sizeof(subsys->model);
3364
3365         if (!uuid_is_null(&ids->uuid))
3366                 return sprintf(buf, "uuid.%pU\n", &ids->uuid);
3367
3368         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3369                 return sprintf(buf, "eui.%16phN\n", ids->nguid);
3370
3371         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3372                 return sprintf(buf, "eui.%8phN\n", ids->eui64);
3373
3374         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3375                                   subsys->serial[serial_len - 1] == '\0'))
3376                 serial_len--;
3377         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3378                                  subsys->model[model_len - 1] == '\0'))
3379                 model_len--;
3380
3381         return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3382                 serial_len, subsys->serial, model_len, subsys->model,
3383                 head->ns_id);
3384 }
3385 static DEVICE_ATTR_RO(wwid);
3386
3387 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3388                 char *buf)
3389 {
3390         return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3391 }
3392 static DEVICE_ATTR_RO(nguid);
3393
3394 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3395                 char *buf)
3396 {
3397         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3398
3399         /* For backward compatibility expose the NGUID to userspace if
3400          * we have no UUID set
3401          */
3402         if (uuid_is_null(&ids->uuid)) {
3403                 printk_ratelimited(KERN_WARNING
3404                                    "No UUID available providing old NGUID\n");
3405                 return sprintf(buf, "%pU\n", ids->nguid);
3406         }
3407         return sprintf(buf, "%pU\n", &ids->uuid);
3408 }
3409 static DEVICE_ATTR_RO(uuid);
3410
3411 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3412                 char *buf)
3413 {
3414         return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3415 }
3416 static DEVICE_ATTR_RO(eui);
3417
3418 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3419                 char *buf)
3420 {
3421         return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3422 }
3423 static DEVICE_ATTR_RO(nsid);
3424
3425 static struct attribute *nvme_ns_id_attrs[] = {
3426         &dev_attr_wwid.attr,
3427         &dev_attr_uuid.attr,
3428         &dev_attr_nguid.attr,
3429         &dev_attr_eui.attr,
3430         &dev_attr_nsid.attr,
3431 #ifdef CONFIG_NVME_MULTIPATH
3432         &dev_attr_ana_grpid.attr,
3433         &dev_attr_ana_state.attr,
3434 #endif
3435         NULL,
3436 };
3437
3438 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3439                 struct attribute *a, int n)
3440 {
3441         struct device *dev = container_of(kobj, struct device, kobj);
3442         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3443
3444         if (a == &dev_attr_uuid.attr) {
3445                 if (uuid_is_null(&ids->uuid) &&
3446                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3447                         return 0;
3448         }
3449         if (a == &dev_attr_nguid.attr) {
3450                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3451                         return 0;
3452         }
3453         if (a == &dev_attr_eui.attr) {
3454                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3455                         return 0;
3456         }
3457 #ifdef CONFIG_NVME_MULTIPATH
3458         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3459                 if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3460                         return 0;
3461                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3462                         return 0;
3463         }
3464 #endif
3465         return a->mode;
3466 }
3467
3468 static const struct attribute_group nvme_ns_id_attr_group = {
3469         .attrs          = nvme_ns_id_attrs,
3470         .is_visible     = nvme_ns_id_attrs_are_visible,
3471 };
3472
3473 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3474         &nvme_ns_id_attr_group,
3475 #ifdef CONFIG_NVM
3476         &nvme_nvm_attr_group,
3477 #endif
3478         NULL,
3479 };
3480
3481 #define nvme_show_str_function(field)                                           \
3482 static ssize_t  field##_show(struct device *dev,                                \
3483                             struct device_attribute *attr, char *buf)           \
3484 {                                                                               \
3485         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3486         return sprintf(buf, "%.*s\n",                                           \
3487                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3488 }                                                                               \
3489 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3490
3491 nvme_show_str_function(model);
3492 nvme_show_str_function(serial);
3493 nvme_show_str_function(firmware_rev);
3494
3495 #define nvme_show_int_function(field)                                           \
3496 static ssize_t  field##_show(struct device *dev,                                \
3497                             struct device_attribute *attr, char *buf)           \
3498 {                                                                               \
3499         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3500         return sprintf(buf, "%d\n", ctrl->field);       \
3501 }                                                                               \
3502 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3503
3504 nvme_show_int_function(cntlid);
3505 nvme_show_int_function(numa_node);
3506 nvme_show_int_function(queue_count);
3507 nvme_show_int_function(sqsize);
3508
3509 static ssize_t nvme_sysfs_delete(struct device *dev,
3510                                 struct device_attribute *attr, const char *buf,
3511                                 size_t count)
3512 {
3513         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3514
3515         if (device_remove_file_self(dev, attr))
3516                 nvme_delete_ctrl_sync(ctrl);
3517         return count;
3518 }
3519 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3520
3521 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3522                                          struct device_attribute *attr,
3523                                          char *buf)
3524 {
3525         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3526
3527         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3528 }
3529 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3530
3531 static ssize_t nvme_sysfs_show_state(struct device *dev,
3532                                      struct device_attribute *attr,
3533                                      char *buf)
3534 {
3535         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3536         static const char *const state_name[] = {
3537                 [NVME_CTRL_NEW]         = "new",
3538                 [NVME_CTRL_LIVE]        = "live",
3539                 [NVME_CTRL_RESETTING]   = "resetting",
3540                 [NVME_CTRL_CONNECTING]  = "connecting",
3541                 [NVME_CTRL_DELETING]    = "deleting",
3542                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3543                 [NVME_CTRL_DEAD]        = "dead",
3544         };
3545
3546         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3547             state_name[ctrl->state])
3548                 return sprintf(buf, "%s\n", state_name[ctrl->state]);
3549
3550         return sprintf(buf, "unknown state\n");
3551 }
3552
3553 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3554
3555 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3556                                          struct device_attribute *attr,
3557                                          char *buf)
3558 {
3559         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3560
3561         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3562 }
3563 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3564
3565 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3566                                         struct device_attribute *attr,
3567                                         char *buf)
3568 {
3569         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3570
3571         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
3572 }
3573 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3574
3575 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3576                                         struct device_attribute *attr,
3577                                         char *buf)
3578 {
3579         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3580
3581         return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
3582 }
3583 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3584
3585 static ssize_t nvme_sysfs_show_address(struct device *dev,
3586                                          struct device_attribute *attr,
3587                                          char *buf)
3588 {
3589         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3590
3591         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3592 }
3593 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3594
3595 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3596                 struct device_attribute *attr, char *buf)
3597 {
3598         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3599         struct nvmf_ctrl_options *opts = ctrl->opts;
3600
3601         if (ctrl->opts->max_reconnects == -1)
3602                 return sprintf(buf, "off\n");
3603         return sprintf(buf, "%d\n",
3604                         opts->max_reconnects * opts->reconnect_delay);
3605 }
3606
3607 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3608                 struct device_attribute *attr, const char *buf, size_t count)
3609 {
3610         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3611         struct nvmf_ctrl_options *opts = ctrl->opts;
3612         int ctrl_loss_tmo, err;
3613
3614         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3615         if (err)
3616                 return -EINVAL;
3617
3618         else if (ctrl_loss_tmo < 0)
3619                 opts->max_reconnects = -1;
3620         else
3621                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3622                                                 opts->reconnect_delay);
3623         return count;
3624 }
3625 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3626         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3627
3628 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3629                 struct device_attribute *attr, char *buf)
3630 {
3631         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3632
3633         if (ctrl->opts->reconnect_delay == -1)
3634                 return sprintf(buf, "off\n");
3635         return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay);
3636 }
3637
3638 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3639                 struct device_attribute *attr, const char *buf, size_t count)
3640 {
3641         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3642         unsigned int v;
3643         int err;
3644
3645         err = kstrtou32(buf, 10, &v);
3646         if (err)
3647                 return err;
3648
3649         ctrl->opts->reconnect_delay = v;
3650         return count;
3651 }
3652 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3653         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3654
3655 static struct attribute *nvme_dev_attrs[] = {
3656         &dev_attr_reset_controller.attr,
3657         &dev_attr_rescan_controller.attr,
3658         &dev_attr_model.attr,
3659         &dev_attr_serial.attr,
3660         &dev_attr_firmware_rev.attr,
3661         &dev_attr_cntlid.attr,
3662         &dev_attr_delete_controller.attr,
3663         &dev_attr_transport.attr,
3664         &dev_attr_subsysnqn.attr,
3665         &dev_attr_address.attr,
3666         &dev_attr_state.attr,
3667         &dev_attr_numa_node.attr,
3668         &dev_attr_queue_count.attr,
3669         &dev_attr_sqsize.attr,
3670         &dev_attr_hostnqn.attr,
3671         &dev_attr_hostid.attr,
3672         &dev_attr_ctrl_loss_tmo.attr,
3673         &dev_attr_reconnect_delay.attr,
3674         NULL
3675 };
3676
3677 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3678                 struct attribute *a, int n)
3679 {
3680         struct device *dev = container_of(kobj, struct device, kobj);
3681         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3682
3683         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3684                 return 0;
3685         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3686                 return 0;
3687         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3688                 return 0;
3689         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3690                 return 0;
3691         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3692                 return 0;
3693         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3694                 return 0;
3695
3696         return a->mode;
3697 }
3698
3699 static struct attribute_group nvme_dev_attrs_group = {
3700         .attrs          = nvme_dev_attrs,
3701         .is_visible     = nvme_dev_attrs_are_visible,
3702 };
3703
3704 static const struct attribute_group *nvme_dev_attr_groups[] = {
3705         &nvme_dev_attrs_group,
3706         NULL,
3707 };
3708
3709 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
3710                 unsigned nsid)
3711 {
3712         struct nvme_ns_head *h;
3713
3714         lockdep_assert_held(&subsys->lock);
3715
3716         list_for_each_entry(h, &subsys->nsheads, entry) {
3717                 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3718                         return h;
3719         }
3720
3721         return NULL;
3722 }
3723
3724 static int __nvme_check_ids(struct nvme_subsystem *subsys,
3725                 struct nvme_ns_head *new)
3726 {
3727         struct nvme_ns_head *h;
3728
3729         lockdep_assert_held(&subsys->lock);
3730
3731         list_for_each_entry(h, &subsys->nsheads, entry) {
3732                 if (nvme_ns_ids_valid(&new->ids) &&
3733                     nvme_ns_ids_equal(&new->ids, &h->ids))
3734                         return -EINVAL;
3735         }
3736
3737         return 0;
3738 }
3739
3740 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3741                 unsigned nsid, struct nvme_ns_ids *ids)
3742 {
3743         struct nvme_ns_head *head;
3744         size_t size = sizeof(*head);
3745         int ret = -ENOMEM;
3746
3747 #ifdef CONFIG_NVME_MULTIPATH
3748         size += num_possible_nodes() * sizeof(struct nvme_ns *);
3749 #endif
3750
3751         head = kzalloc(size, GFP_KERNEL);
3752         if (!head)
3753                 goto out;
3754         ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3755         if (ret < 0)
3756                 goto out_free_head;
3757         head->instance = ret;
3758         INIT_LIST_HEAD(&head->list);
3759         ret = init_srcu_struct(&head->srcu);
3760         if (ret)
3761                 goto out_ida_remove;
3762         head->subsys = ctrl->subsys;
3763         head->ns_id = nsid;
3764         head->ids = *ids;
3765         kref_init(&head->ref);
3766
3767         ret = __nvme_check_ids(ctrl->subsys, head);
3768         if (ret) {
3769                 dev_err(ctrl->device,
3770                         "duplicate IDs for nsid %d\n", nsid);
3771                 goto out_cleanup_srcu;
3772         }
3773
3774         if (head->ids.csi) {
3775                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3776                 if (ret)
3777                         goto out_cleanup_srcu;
3778         } else
3779                 head->effects = ctrl->effects;
3780
3781         ret = nvme_mpath_alloc_disk(ctrl, head);
3782         if (ret)
3783                 goto out_cleanup_srcu;
3784
3785         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3786
3787         kref_get(&ctrl->subsys->ref);
3788
3789         return head;
3790 out_cleanup_srcu:
3791         cleanup_srcu_struct(&head->srcu);
3792 out_ida_remove:
3793         ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3794 out_free_head:
3795         kfree(head);
3796 out:
3797         if (ret > 0)
3798                 ret = blk_status_to_errno(nvme_error_status(ret));
3799         return ERR_PTR(ret);
3800 }
3801
3802 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3803                 struct nvme_id_ns *id)
3804 {
3805         struct nvme_ctrl *ctrl = ns->ctrl;
3806         bool is_shared = id->nmic & NVME_NS_NMIC_SHARED;
3807         struct nvme_ns_head *head = NULL;
3808         struct nvme_ns_ids ids;
3809         int ret = 0;
3810
3811         ret = nvme_report_ns_ids(ctrl, nsid, id, &ids);
3812         if (ret) {
3813                 if (ret < 0)
3814                         return ret;
3815                 return blk_status_to_errno(nvme_error_status(ret));
3816         }
3817
3818         mutex_lock(&ctrl->subsys->lock);
3819         head = nvme_find_ns_head(ctrl->subsys, nsid);
3820         if (!head) {
3821                 head = nvme_alloc_ns_head(ctrl, nsid, &ids);
3822                 if (IS_ERR(head)) {
3823                         ret = PTR_ERR(head);
3824                         goto out_unlock;
3825                 }
3826                 head->shared = is_shared;
3827         } else {
3828                 ret = -EINVAL;
3829                 if (!is_shared || !head->shared) {
3830                         dev_err(ctrl->device,
3831                                 "Duplicate unshared namespace %d\n", nsid);
3832                         goto out_put_ns_head;
3833                 }
3834                 if (!nvme_ns_ids_equal(&head->ids, &ids)) {
3835                         dev_err(ctrl->device,
3836                                 "IDs don't match for shared namespace %d\n",
3837                                         nsid);
3838                         goto out_put_ns_head;
3839                 }
3840         }
3841
3842         list_add_tail(&ns->siblings, &head->list);
3843         ns->head = head;
3844         mutex_unlock(&ctrl->subsys->lock);
3845         return 0;
3846
3847 out_put_ns_head:
3848         nvme_put_ns_head(head);
3849 out_unlock:
3850         mutex_unlock(&ctrl->subsys->lock);
3851         return ret;
3852 }
3853
3854 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3855 {
3856         struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3857         struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3858
3859         return nsa->head->ns_id - nsb->head->ns_id;
3860 }
3861
3862 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3863 {
3864         struct nvme_ns *ns, *ret = NULL;
3865
3866         down_read(&ctrl->namespaces_rwsem);
3867         list_for_each_entry(ns, &ctrl->namespaces, list) {
3868                 if (ns->head->ns_id == nsid) {
3869                         if (!kref_get_unless_zero(&ns->kref))
3870                                 continue;
3871                         ret = ns;
3872                         break;
3873                 }
3874                 if (ns->head->ns_id > nsid)
3875                         break;
3876         }
3877         up_read(&ctrl->namespaces_rwsem);
3878         return ret;
3879 }
3880 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3881
3882 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3883 {
3884         struct nvme_ns *ns;
3885         struct gendisk *disk;
3886         struct nvme_id_ns *id;
3887         char disk_name[DISK_NAME_LEN];
3888         int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3889
3890         if (nvme_identify_ns(ctrl, nsid, &id))
3891                 return;
3892
3893         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3894         if (!ns)
3895                 goto out_free_id;
3896
3897         ns->queue = blk_mq_init_queue(ctrl->tagset);
3898         if (IS_ERR(ns->queue))
3899                 goto out_free_ns;
3900
3901         if (ctrl->opts && ctrl->opts->data_digest)
3902                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3903
3904         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3905         if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3906                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3907
3908         ns->queue->queuedata = ns;
3909         ns->ctrl = ctrl;
3910
3911         kref_init(&ns->kref);
3912         ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
3913
3914         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
3915         nvme_set_queue_limits(ctrl, ns->queue);
3916
3917         ret = nvme_init_ns_head(ns, nsid, id);
3918         if (ret)
3919                 goto out_free_queue;
3920         nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3921
3922         disk = alloc_disk_node(0, node);
3923         if (!disk)
3924                 goto out_unlink_ns;
3925
3926         disk->fops = &nvme_fops;
3927         disk->private_data = ns;
3928         disk->queue = ns->queue;
3929         disk->flags = flags;
3930         memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3931         ns->disk = disk;
3932
3933         if (nvme_update_ns_info(ns, id))
3934                 goto out_put_disk;
3935         if (blk_queue_is_zoned(ns->queue) && nvme_revalidate_zones(ns))
3936                 goto out_put_disk;
3937
3938         if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3939                 ret = nvme_nvm_register(ns, disk_name, node);
3940                 if (ret) {
3941                         dev_warn(ctrl->device, "LightNVM init failure\n");
3942                         goto out_put_disk;
3943                 }
3944         }
3945
3946         down_write(&ctrl->namespaces_rwsem);
3947         list_add_tail(&ns->list, &ctrl->namespaces);
3948         up_write(&ctrl->namespaces_rwsem);
3949
3950         nvme_get_ctrl(ctrl);
3951
3952         device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3953
3954         nvme_mpath_add_disk(ns, id);
3955         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3956         kfree(id);
3957
3958         return;
3959  out_put_disk:
3960         /* prevent double queue cleanup */
3961         ns->disk->queue = NULL;
3962         put_disk(ns->disk);
3963  out_unlink_ns:
3964         mutex_lock(&ctrl->subsys->lock);
3965         list_del_rcu(&ns->siblings);
3966         if (list_empty(&ns->head->list))
3967                 list_del_init(&ns->head->entry);
3968         mutex_unlock(&ctrl->subsys->lock);
3969         nvme_put_ns_head(ns->head);
3970  out_free_queue:
3971         blk_cleanup_queue(ns->queue);
3972  out_free_ns:
3973         kfree(ns);
3974  out_free_id:
3975         kfree(id);
3976 }
3977
3978 static void nvme_ns_remove(struct nvme_ns *ns)
3979 {
3980         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3981                 return;
3982
3983         nvme_fault_inject_fini(&ns->fault_inject);
3984
3985         mutex_lock(&ns->ctrl->subsys->lock);
3986         list_del_rcu(&ns->siblings);
3987         if (list_empty(&ns->head->list))
3988                 list_del_init(&ns->head->entry);
3989         mutex_unlock(&ns->ctrl->subsys->lock);
3990
3991         synchronize_rcu(); /* guarantee not available in head->list */
3992         nvme_mpath_clear_current_path(ns);
3993         synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3994
3995         if (ns->disk->flags & GENHD_FL_UP) {
3996                 del_gendisk(ns->disk);
3997                 blk_cleanup_queue(ns->queue);
3998                 if (blk_get_integrity(ns->disk))
3999                         blk_integrity_unregister(ns->disk);
4000         }
4001
4002         down_write(&ns->ctrl->namespaces_rwsem);
4003         list_del_init(&ns->list);
4004         up_write(&ns->ctrl->namespaces_rwsem);
4005
4006         nvme_mpath_check_last_path(ns);
4007         nvme_put_ns(ns);
4008 }
4009
4010 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4011 {
4012         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4013
4014         if (ns) {
4015                 nvme_ns_remove(ns);
4016                 nvme_put_ns(ns);
4017         }
4018 }
4019
4020 static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4021 {
4022         struct nvme_ns *ns;
4023         int ret;
4024
4025         ns = nvme_find_get_ns(ctrl, nsid);
4026         if (!ns) {
4027                 nvme_alloc_ns(ctrl, nsid);
4028                 return;
4029         }
4030
4031         ret = nvme_validate_ns(ns);
4032         if (!ret && blk_queue_is_zoned(ns->queue))
4033                 ret = nvme_revalidate_zones(ns);
4034         revalidate_disk_size(ns->disk, ret == 0);
4035         if (ret)
4036                 nvme_ns_remove(ns);
4037         nvme_put_ns(ns);
4038 }
4039
4040 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4041                                         unsigned nsid)
4042 {
4043         struct nvme_ns *ns, *next;
4044         LIST_HEAD(rm_list);
4045
4046         down_write(&ctrl->namespaces_rwsem);
4047         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4048                 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4049                         list_move_tail(&ns->list, &rm_list);
4050         }
4051         up_write(&ctrl->namespaces_rwsem);
4052
4053         list_for_each_entry_safe(ns, next, &rm_list, list)
4054                 nvme_ns_remove(ns);
4055
4056 }
4057
4058 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4059 {
4060         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4061         __le32 *ns_list;
4062         u32 prev = 0;
4063         int ret = 0, i;
4064
4065         if (nvme_ctrl_limited_cns(ctrl))
4066                 return -EOPNOTSUPP;
4067
4068         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4069         if (!ns_list)
4070                 return -ENOMEM;
4071
4072         for (;;) {
4073                 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
4074                 if (ret)
4075                         goto free;
4076
4077                 for (i = 0; i < nr_entries; i++) {
4078                         u32 nsid = le32_to_cpu(ns_list[i]);
4079
4080                         if (!nsid)      /* end of the list? */
4081                                 goto out;
4082                         nvme_validate_or_alloc_ns(ctrl, nsid);
4083                         while (++prev < nsid)
4084                                 nvme_ns_remove_by_nsid(ctrl, prev);
4085                 }
4086         }
4087  out:
4088         nvme_remove_invalid_namespaces(ctrl, prev);
4089  free:
4090         kfree(ns_list);
4091         return ret;
4092 }
4093
4094 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4095 {
4096         struct nvme_id_ctrl *id;
4097         u32 nn, i;
4098
4099         if (nvme_identify_ctrl(ctrl, &id))
4100                 return;
4101         nn = le32_to_cpu(id->nn);
4102         kfree(id);
4103
4104         for (i = 1; i <= nn; i++)
4105                 nvme_validate_or_alloc_ns(ctrl, i);
4106
4107         nvme_remove_invalid_namespaces(ctrl, nn);
4108 }
4109
4110 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4111 {
4112         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4113         __le32 *log;
4114         int error;
4115
4116         log = kzalloc(log_size, GFP_KERNEL);
4117         if (!log)
4118                 return;
4119
4120         /*
4121          * We need to read the log to clear the AEN, but we don't want to rely
4122          * on it for the changed namespace information as userspace could have
4123          * raced with us in reading the log page, which could cause us to miss
4124          * updates.
4125          */
4126         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4127                         NVME_CSI_NVM, log, log_size, 0);
4128         if (error)
4129                 dev_warn(ctrl->device,
4130                         "reading changed ns log failed: %d\n", error);
4131
4132         kfree(log);
4133 }
4134
4135 static void nvme_scan_work(struct work_struct *work)
4136 {
4137         struct nvme_ctrl *ctrl =
4138                 container_of(work, struct nvme_ctrl, scan_work);
4139
4140         /* No tagset on a live ctrl means IO queues could not created */
4141         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4142                 return;
4143
4144         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4145                 dev_info(ctrl->device, "rescanning namespaces.\n");
4146                 nvme_clear_changed_ns_log(ctrl);
4147         }
4148
4149         mutex_lock(&ctrl->scan_lock);
4150         if (nvme_scan_ns_list(ctrl) != 0)
4151                 nvme_scan_ns_sequential(ctrl);
4152         mutex_unlock(&ctrl->scan_lock);
4153
4154         down_write(&ctrl->namespaces_rwsem);
4155         list_sort(NULL, &ctrl->namespaces, ns_cmp);
4156         up_write(&ctrl->namespaces_rwsem);
4157 }
4158
4159 /*
4160  * This function iterates the namespace list unlocked to allow recovery from
4161  * controller failure. It is up to the caller to ensure the namespace list is
4162  * not modified by scan work while this function is executing.
4163  */
4164 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4165 {
4166         struct nvme_ns *ns, *next;
4167         LIST_HEAD(ns_list);
4168
4169         /*
4170          * make sure to requeue I/O to all namespaces as these
4171          * might result from the scan itself and must complete
4172          * for the scan_work to make progress
4173          */
4174         nvme_mpath_clear_ctrl_paths(ctrl);
4175
4176         /* prevent racing with ns scanning */
4177         flush_work(&ctrl->scan_work);
4178
4179         /*
4180          * The dead states indicates the controller was not gracefully
4181          * disconnected. In that case, we won't be able to flush any data while
4182          * removing the namespaces' disks; fail all the queues now to avoid
4183          * potentially having to clean up the failed sync later.
4184          */
4185         if (ctrl->state == NVME_CTRL_DEAD)
4186                 nvme_kill_queues(ctrl);
4187
4188         /* this is a no-op when called from the controller reset handler */
4189         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4190
4191         down_write(&ctrl->namespaces_rwsem);
4192         list_splice_init(&ctrl->namespaces, &ns_list);
4193         up_write(&ctrl->namespaces_rwsem);
4194
4195         list_for_each_entry_safe(ns, next, &ns_list, list)
4196                 nvme_ns_remove(ns);
4197 }
4198 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4199
4200 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4201 {
4202         struct nvme_ctrl *ctrl =
4203                 container_of(dev, struct nvme_ctrl, ctrl_device);
4204         struct nvmf_ctrl_options *opts = ctrl->opts;
4205         int ret;
4206
4207         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4208         if (ret)
4209                 return ret;
4210
4211         if (opts) {
4212                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4213                 if (ret)
4214                         return ret;
4215
4216                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4217                                 opts->trsvcid ?: "none");
4218                 if (ret)
4219                         return ret;
4220
4221                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4222                                 opts->host_traddr ?: "none");
4223         }
4224         return ret;
4225 }
4226
4227 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4228 {
4229         char *envp[2] = { NULL, NULL };
4230         u32 aen_result = ctrl->aen_result;
4231
4232         ctrl->aen_result = 0;
4233         if (!aen_result)
4234                 return;
4235
4236         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4237         if (!envp[0])
4238                 return;
4239         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4240         kfree(envp[0]);
4241 }
4242
4243 static void nvme_async_event_work(struct work_struct *work)
4244 {
4245         struct nvme_ctrl *ctrl =
4246                 container_of(work, struct nvme_ctrl, async_event_work);
4247
4248         nvme_aen_uevent(ctrl);
4249         ctrl->ops->submit_async_event(ctrl);
4250 }
4251
4252 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4253 {
4254
4255         u32 csts;
4256
4257         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4258                 return false;
4259
4260         if (csts == ~0)
4261                 return false;
4262
4263         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4264 }
4265
4266 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4267 {
4268         struct nvme_fw_slot_info_log *log;
4269
4270         log = kmalloc(sizeof(*log), GFP_KERNEL);
4271         if (!log)
4272                 return;
4273
4274         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4275                         log, sizeof(*log), 0))
4276                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4277         kfree(log);
4278 }
4279
4280 static void nvme_fw_act_work(struct work_struct *work)
4281 {
4282         struct nvme_ctrl *ctrl = container_of(work,
4283                                 struct nvme_ctrl, fw_act_work);
4284         unsigned long fw_act_timeout;
4285
4286         if (ctrl->mtfa)
4287                 fw_act_timeout = jiffies +
4288                                 msecs_to_jiffies(ctrl->mtfa * 100);
4289         else
4290                 fw_act_timeout = jiffies +
4291                                 msecs_to_jiffies(admin_timeout * 1000);
4292
4293         nvme_stop_queues(ctrl);
4294         while (nvme_ctrl_pp_status(ctrl)) {
4295                 if (time_after(jiffies, fw_act_timeout)) {
4296                         dev_warn(ctrl->device,
4297                                 "Fw activation timeout, reset controller\n");
4298                         nvme_try_sched_reset(ctrl);
4299                         return;
4300                 }
4301                 msleep(100);
4302         }
4303
4304         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4305                 return;
4306
4307         nvme_start_queues(ctrl);
4308         /* read FW slot information to clear the AER */
4309         nvme_get_fw_slot_info(ctrl);
4310 }
4311
4312 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4313 {
4314         u32 aer_notice_type = (result & 0xff00) >> 8;
4315
4316         trace_nvme_async_event(ctrl, aer_notice_type);
4317
4318         switch (aer_notice_type) {
4319         case NVME_AER_NOTICE_NS_CHANGED:
4320                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4321                 nvme_queue_scan(ctrl);
4322                 break;
4323         case NVME_AER_NOTICE_FW_ACT_STARTING:
4324                 /*
4325                  * We are (ab)using the RESETTING state to prevent subsequent
4326                  * recovery actions from interfering with the controller's
4327                  * firmware activation.
4328                  */
4329                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4330                         queue_work(nvme_wq, &ctrl->fw_act_work);
4331                 break;
4332 #ifdef CONFIG_NVME_MULTIPATH
4333         case NVME_AER_NOTICE_ANA:
4334                 if (!ctrl->ana_log_buf)
4335                         break;
4336                 queue_work(nvme_wq, &ctrl->ana_work);
4337                 break;
4338 #endif
4339         case NVME_AER_NOTICE_DISC_CHANGED:
4340                 ctrl->aen_result = result;
4341                 break;
4342         default:
4343                 dev_warn(ctrl->device, "async event result %08x\n", result);
4344         }
4345 }
4346
4347 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4348                 volatile union nvme_result *res)
4349 {
4350         u32 result = le32_to_cpu(res->u32);
4351         u32 aer_type = result & 0x07;
4352
4353         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4354                 return;
4355
4356         switch (aer_type) {
4357         case NVME_AER_NOTICE:
4358                 nvme_handle_aen_notice(ctrl, result);
4359                 break;
4360         case NVME_AER_ERROR:
4361         case NVME_AER_SMART:
4362         case NVME_AER_CSS:
4363         case NVME_AER_VS:
4364                 trace_nvme_async_event(ctrl, aer_type);
4365                 ctrl->aen_result = result;
4366                 break;
4367         default:
4368                 break;
4369         }
4370         queue_work(nvme_wq, &ctrl->async_event_work);
4371 }
4372 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4373
4374 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4375 {
4376         nvme_mpath_stop(ctrl);
4377         nvme_stop_keep_alive(ctrl);
4378         flush_work(&ctrl->async_event_work);
4379         cancel_work_sync(&ctrl->fw_act_work);
4380 }
4381 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4382
4383 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4384 {
4385         nvme_start_keep_alive(ctrl);
4386
4387         nvme_enable_aen(ctrl);
4388
4389         if (ctrl->queue_count > 1) {
4390                 nvme_queue_scan(ctrl);
4391                 nvme_start_queues(ctrl);
4392         }
4393 }
4394 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4395
4396 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4397 {
4398         nvme_fault_inject_fini(&ctrl->fault_inject);
4399         dev_pm_qos_hide_latency_tolerance(ctrl->device);
4400         cdev_device_del(&ctrl->cdev, ctrl->device);
4401         nvme_put_ctrl(ctrl);
4402 }
4403 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4404
4405 static void nvme_free_ctrl(struct device *dev)
4406 {
4407         struct nvme_ctrl *ctrl =
4408                 container_of(dev, struct nvme_ctrl, ctrl_device);
4409         struct nvme_subsystem *subsys = ctrl->subsys;
4410
4411         if (!subsys || ctrl->instance != subsys->instance)
4412                 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4413
4414         xa_destroy(&ctrl->cels);
4415
4416         nvme_mpath_uninit(ctrl);
4417         __free_page(ctrl->discard_page);
4418
4419         if (subsys) {
4420                 mutex_lock(&nvme_subsystems_lock);
4421                 list_del(&ctrl->subsys_entry);
4422                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4423                 mutex_unlock(&nvme_subsystems_lock);
4424         }
4425
4426         ctrl->ops->free_ctrl(ctrl);
4427
4428         if (subsys)
4429                 nvme_put_subsystem(subsys);
4430 }
4431
4432 /*
4433  * Initialize a NVMe controller structures.  This needs to be called during
4434  * earliest initialization so that we have the initialized structured around
4435  * during probing.
4436  */
4437 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4438                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4439 {
4440         int ret;
4441
4442         ctrl->state = NVME_CTRL_NEW;
4443         spin_lock_init(&ctrl->lock);
4444         mutex_init(&ctrl->scan_lock);
4445         INIT_LIST_HEAD(&ctrl->namespaces);
4446         xa_init(&ctrl->cels);
4447         init_rwsem(&ctrl->namespaces_rwsem);
4448         ctrl->dev = dev;
4449         ctrl->ops = ops;
4450         ctrl->quirks = quirks;
4451         ctrl->numa_node = NUMA_NO_NODE;
4452         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4453         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4454         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4455         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4456         init_waitqueue_head(&ctrl->state_wq);
4457
4458         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4459         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4460         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4461
4462         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4463                         PAGE_SIZE);
4464         ctrl->discard_page = alloc_page(GFP_KERNEL);
4465         if (!ctrl->discard_page) {
4466                 ret = -ENOMEM;
4467                 goto out;
4468         }
4469
4470         ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4471         if (ret < 0)
4472                 goto out;
4473         ctrl->instance = ret;
4474
4475         device_initialize(&ctrl->ctrl_device);
4476         ctrl->device = &ctrl->ctrl_device;
4477         ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4478         ctrl->device->class = nvme_class;
4479         ctrl->device->parent = ctrl->dev;
4480         ctrl->device->groups = nvme_dev_attr_groups;
4481         ctrl->device->release = nvme_free_ctrl;
4482         dev_set_drvdata(ctrl->device, ctrl);
4483         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4484         if (ret)
4485                 goto out_release_instance;
4486
4487         nvme_get_ctrl(ctrl);
4488         cdev_init(&ctrl->cdev, &nvme_dev_fops);
4489         ctrl->cdev.owner = ops->module;
4490         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4491         if (ret)
4492                 goto out_free_name;
4493
4494         /*
4495          * Initialize latency tolerance controls.  The sysfs files won't
4496          * be visible to userspace unless the device actually supports APST.
4497          */
4498         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4499         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4500                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4501
4502         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4503
4504         return 0;
4505 out_free_name:
4506         nvme_put_ctrl(ctrl);
4507         kfree_const(ctrl->device->kobj.name);
4508 out_release_instance:
4509         ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4510 out:
4511         if (ctrl->discard_page)
4512                 __free_page(ctrl->discard_page);
4513         return ret;
4514 }
4515 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4516
4517 /**
4518  * nvme_kill_queues(): Ends all namespace queues
4519  * @ctrl: the dead controller that needs to end
4520  *
4521  * Call this function when the driver determines it is unable to get the
4522  * controller in a state capable of servicing IO.
4523  */
4524 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4525 {
4526         struct nvme_ns *ns;
4527
4528         down_read(&ctrl->namespaces_rwsem);
4529
4530         /* Forcibly unquiesce queues to avoid blocking dispatch */
4531         if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4532                 blk_mq_unquiesce_queue(ctrl->admin_q);
4533
4534         list_for_each_entry(ns, &ctrl->namespaces, list)
4535                 nvme_set_queue_dying(ns);
4536
4537         up_read(&ctrl->namespaces_rwsem);
4538 }
4539 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4540
4541 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4542 {
4543         struct nvme_ns *ns;
4544
4545         down_read(&ctrl->namespaces_rwsem);
4546         list_for_each_entry(ns, &ctrl->namespaces, list)
4547                 blk_mq_unfreeze_queue(ns->queue);
4548         up_read(&ctrl->namespaces_rwsem);
4549 }
4550 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4551
4552 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4553 {
4554         struct nvme_ns *ns;
4555
4556         down_read(&ctrl->namespaces_rwsem);
4557         list_for_each_entry(ns, &ctrl->namespaces, list) {
4558                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4559                 if (timeout <= 0)
4560                         break;
4561         }
4562         up_read(&ctrl->namespaces_rwsem);
4563         return timeout;
4564 }
4565 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4566
4567 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4568 {
4569         struct nvme_ns *ns;
4570
4571         down_read(&ctrl->namespaces_rwsem);
4572         list_for_each_entry(ns, &ctrl->namespaces, list)
4573                 blk_mq_freeze_queue_wait(ns->queue);
4574         up_read(&ctrl->namespaces_rwsem);
4575 }
4576 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4577
4578 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4579 {
4580         struct nvme_ns *ns;
4581
4582         down_read(&ctrl->namespaces_rwsem);
4583         list_for_each_entry(ns, &ctrl->namespaces, list)
4584                 blk_freeze_queue_start(ns->queue);
4585         up_read(&ctrl->namespaces_rwsem);
4586 }
4587 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4588
4589 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4590 {
4591         struct nvme_ns *ns;
4592
4593         down_read(&ctrl->namespaces_rwsem);
4594         list_for_each_entry(ns, &ctrl->namespaces, list)
4595                 blk_mq_quiesce_queue(ns->queue);
4596         up_read(&ctrl->namespaces_rwsem);
4597 }
4598 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4599
4600 void nvme_start_queues(struct nvme_ctrl *ctrl)
4601 {
4602         struct nvme_ns *ns;
4603
4604         down_read(&ctrl->namespaces_rwsem);
4605         list_for_each_entry(ns, &ctrl->namespaces, list)
4606                 blk_mq_unquiesce_queue(ns->queue);
4607         up_read(&ctrl->namespaces_rwsem);
4608 }
4609 EXPORT_SYMBOL_GPL(nvme_start_queues);
4610
4611
4612 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4613 {
4614         struct nvme_ns *ns;
4615
4616         down_read(&ctrl->namespaces_rwsem);
4617         list_for_each_entry(ns, &ctrl->namespaces, list)
4618                 blk_sync_queue(ns->queue);
4619         up_read(&ctrl->namespaces_rwsem);
4620
4621         if (ctrl->admin_q)
4622                 blk_sync_queue(ctrl->admin_q);
4623 }
4624 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4625
4626 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4627 {
4628         if (file->f_op != &nvme_dev_fops)
4629                 return NULL;
4630         return file->private_data;
4631 }
4632 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
4633
4634 /*
4635  * Check we didn't inadvertently grow the command structure sizes:
4636  */
4637 static inline void _nvme_check_size(void)
4638 {
4639         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4640         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4641         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4642         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4643         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4644         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4645         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4646         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4647         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4648         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4649         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4650         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4651         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4652         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4653         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4654         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4655         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4656         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4657         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4658 }
4659
4660
4661 static int __init nvme_core_init(void)
4662 {
4663         int result = -ENOMEM;
4664
4665         _nvme_check_size();
4666
4667         nvme_wq = alloc_workqueue("nvme-wq",
4668                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4669         if (!nvme_wq)
4670                 goto out;
4671
4672         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4673                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4674         if (!nvme_reset_wq)
4675                 goto destroy_wq;
4676
4677         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4678                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4679         if (!nvme_delete_wq)
4680                 goto destroy_reset_wq;
4681
4682         result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4683         if (result < 0)
4684                 goto destroy_delete_wq;
4685
4686         nvme_class = class_create(THIS_MODULE, "nvme");
4687         if (IS_ERR(nvme_class)) {
4688                 result = PTR_ERR(nvme_class);
4689                 goto unregister_chrdev;
4690         }
4691         nvme_class->dev_uevent = nvme_class_uevent;
4692
4693         nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4694         if (IS_ERR(nvme_subsys_class)) {
4695                 result = PTR_ERR(nvme_subsys_class);
4696                 goto destroy_class;
4697         }
4698         return 0;
4699
4700 destroy_class:
4701         class_destroy(nvme_class);
4702 unregister_chrdev:
4703         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4704 destroy_delete_wq:
4705         destroy_workqueue(nvme_delete_wq);
4706 destroy_reset_wq:
4707         destroy_workqueue(nvme_reset_wq);
4708 destroy_wq:
4709         destroy_workqueue(nvme_wq);
4710 out:
4711         return result;
4712 }
4713
4714 static void __exit nvme_core_exit(void)
4715 {
4716         class_destroy(nvme_subsys_class);
4717         class_destroy(nvme_class);
4718         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4719         destroy_workqueue(nvme_delete_wq);
4720         destroy_workqueue(nvme_reset_wq);
4721         destroy_workqueue(nvme_wq);
4722         ida_destroy(&nvme_instance_ida);
4723 }
4724
4725 MODULE_LICENSE("GPL");
4726 MODULE_VERSION("1.0");
4727 module_init(nvme_core_init);
4728 module_exit(nvme_core_exit);