drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/compat.h>
  10 #include <linux/delay.h>
  11 #include <linux/errno.h>
  12 #include <linux/hdreg.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/backing-dev.h>
  16 #include <linux/list_sort.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <asm/unaligned.h>
  24
  25 #include "nvme.h"
  26 #include "fabrics.h"
  27
  28 #define CREATE_TRACE_POINTS
  29 #include "trace.h"
  30
  31 #define NVME_MINORS             (1U << MINORBITS)
  32
  33 unsigned int admin_timeout = 60;
  34 module_param(admin_timeout, uint, 0644);
  35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  36 EXPORT_SYMBOL_GPL(admin_timeout);
  37
  38 unsigned int nvme_io_timeout = 30;
  39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  42
  43 static unsigned char shutdown_timeout = 5;
  44 module_param(shutdown_timeout, byte, 0644);
  45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  46
  47 static u8 nvme_max_retries = 5;
  48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  50
  51 static unsigned long default_ps_max_latency_us = 100000;
  52 module_param(default_ps_max_latency_us, ulong, 0644);
  53 MODULE_PARM_DESC(default_ps_max_latency_us,
  54                  "max power saving latency for new devices; use PM QOS to change per device");
  55
  56 static bool force_apst;
  57 module_param(force_apst, bool, 0644);
  58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  59
  60 static bool streams;
  61 module_param(streams, bool, 0644);
  62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
  63
  64 /*
  65  * nvme_wq - hosts nvme related works that are not reset or delete
  66  * nvme_reset_wq - hosts nvme reset works
  67  * nvme_delete_wq - hosts nvme delete works
  68  *
  69  * nvme_wq will host works such as scan, aen handling, fw activation,
  70  * keep-alive, periodic reconnects etc. nvme_reset_wq
  71  * runs reset works which also flush works hosted on nvme_wq for
  72  * serialization purposes. nvme_delete_wq host controller deletion
  73  * works which flush reset works for serialization.
  74  */
  75 struct workqueue_struct *nvme_wq;
  76 EXPORT_SYMBOL_GPL(nvme_wq);
  77
  78 struct workqueue_struct *nvme_reset_wq;
  79 EXPORT_SYMBOL_GPL(nvme_reset_wq);
  80
  81 struct workqueue_struct *nvme_delete_wq;
  82 EXPORT_SYMBOL_GPL(nvme_delete_wq);
  83
  84 static LIST_HEAD(nvme_subsystems);
  85 static DEFINE_MUTEX(nvme_subsystems_lock);
  86
  87 static DEFINE_IDA(nvme_instance_ida);
  88 static dev_t nvme_chr_devt;
  89 static struct class *nvme_class;
  90 static struct class *nvme_subsys_class;
  91
  92 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
  93 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
  94                                            unsigned nsid);
  95
  96 static void nvme_update_bdev_size(struct gendisk *disk)
  97 {
  98         struct block_device *bdev = bdget_disk(disk, 0);
  99
 100         if (bdev) {
 101                 bd_set_nr_sectors(bdev, get_capacity(disk));
 102                 bdput(bdev);
 103         }
 104 }
 105
 106 /*
 107  * Prepare a queue for teardown.
 108  *
 109  * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
 110  * the capacity to 0 after that to avoid blocking dispatchers that may be
 111  * holding bd_butex.  This will end buffered writers dirtying pages that can't
 112  * be synced.
 113  */
 114 static void nvme_set_queue_dying(struct nvme_ns *ns)
 115 {
 116         if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
 117                 return;
 118
 119         blk_set_queue_dying(ns->queue);
 120         blk_mq_unquiesce_queue(ns->queue);
 121
 122         set_capacity(ns->disk, 0);
 123         nvme_update_bdev_size(ns->disk);
 124 }
 125
 126 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
 127 {
 128         /*
 129          * Only new queue scan work when admin and IO queues are both alive
 130          */
 131         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 132                 queue_work(nvme_wq, &ctrl->scan_work);
 133 }
 134
 135 /*
 136  * Use this function to proceed with scheduling reset_work for a controller
 137  * that had previously been set to the resetting state. This is intended for
 138  * code paths that can't be interrupted by other reset attempts. A hot removal
 139  * may prevent this from succeeding.
 140  */
 141 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 142 {
 143         if (ctrl->state != NVME_CTRL_RESETTING)
 144                 return -EBUSY;
 145         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 146                 return -EBUSY;
 147         return 0;
 148 }
 149 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 150
 151 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 152 {
 153         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 154                 return -EBUSY;
 155         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 156                 return -EBUSY;
 157         return 0;
 158 }
 159 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 160
 161 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 162 {
 163         int ret;
 164
 165         ret = nvme_reset_ctrl(ctrl);
 166         if (!ret) {
 167                 flush_work(&ctrl->reset_work);
 168                 if (ctrl->state != NVME_CTRL_LIVE)
 169                         ret = -ENETRESET;
 170         }
 171
 172         return ret;
 173 }
 174 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
 175
 176 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 177 {
 178         dev_info(ctrl->device,
 179                  "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
 180
 181         flush_work(&ctrl->reset_work);
 182         nvme_stop_ctrl(ctrl);
 183         nvme_remove_namespaces(ctrl);
 184         ctrl->ops->delete_ctrl(ctrl);
 185         nvme_uninit_ctrl(ctrl);
 186 }
 187
 188 static void nvme_delete_ctrl_work(struct work_struct *work)
 189 {
 190         struct nvme_ctrl *ctrl =
 191                 container_of(work, struct nvme_ctrl, delete_work);
 192
 193         nvme_do_delete_ctrl(ctrl);
 194 }
 195
 196 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 197 {
 198         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 199                 return -EBUSY;
 200         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 201                 return -EBUSY;
 202         return 0;
 203 }
 204 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 205
 206 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 207 {
 208         /*
 209          * Keep a reference until nvme_do_delete_ctrl() complete,
 210          * since ->delete_ctrl can free the controller.
 211          */
 212         nvme_get_ctrl(ctrl);
 213         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 214                 nvme_do_delete_ctrl(ctrl);
 215         nvme_put_ctrl(ctrl);
 216 }
 217
 218 static blk_status_t nvme_error_status(u16 status)
 219 {
 220         switch (status & 0x7ff) {
 221         case NVME_SC_SUCCESS:
 222                 return BLK_STS_OK;
 223         case NVME_SC_CAP_EXCEEDED:
 224                 return BLK_STS_NOSPC;
 225         case NVME_SC_LBA_RANGE:
 226         case NVME_SC_CMD_INTERRUPTED:
 227         case NVME_SC_NS_NOT_READY:
 228                 return BLK_STS_TARGET;
 229         case NVME_SC_BAD_ATTRIBUTES:
 230         case NVME_SC_ONCS_NOT_SUPPORTED:
 231         case NVME_SC_INVALID_OPCODE:
 232         case NVME_SC_INVALID_FIELD:
 233         case NVME_SC_INVALID_NS:
 234                 return BLK_STS_NOTSUPP;
 235         case NVME_SC_WRITE_FAULT:
 236         case NVME_SC_READ_ERROR:
 237         case NVME_SC_UNWRITTEN_BLOCK:
 238         case NVME_SC_ACCESS_DENIED:
 239         case NVME_SC_READ_ONLY:
 240         case NVME_SC_COMPARE_FAILED:
 241                 return BLK_STS_MEDIUM;
 242         case NVME_SC_GUARD_CHECK:
 243         case NVME_SC_APPTAG_CHECK:
 244         case NVME_SC_REFTAG_CHECK:
 245         case NVME_SC_INVALID_PI:
 246                 return BLK_STS_PROTECTION;
 247         case NVME_SC_RESERVATION_CONFLICT:
 248                 return BLK_STS_NEXUS;
 249         case NVME_SC_HOST_PATH_ERROR:
 250                 return BLK_STS_TRANSPORT;
 251         default:
 252                 return BLK_STS_IOERR;
 253         }
 254 }
 255
 256 static void nvme_retry_req(struct request *req)
 257 {
 258         struct nvme_ns *ns = req->q->queuedata;
 259         unsigned long delay = 0;
 260         u16 crd;
 261
 262         /* The mask and shift result must be <= 3 */
 263         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 264         if (ns && crd)
 265                 delay = ns->ctrl->crdt[crd - 1] * 100;
 266
 267         nvme_req(req)->retries++;
 268         blk_mq_requeue_request(req, false);
 269         blk_mq_delay_kick_requeue_list(req->q, delay);
 270 }
 271
 272 enum nvme_disposition {
 273         COMPLETE,
 274         RETRY,
 275         FAILOVER,
 276 };
 277
 278 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 279 {
 280         if (likely(nvme_req(req)->status == 0))
 281                 return COMPLETE;
 282
 283         if (blk_noretry_request(req) ||
 284             (nvme_req(req)->status & NVME_SC_DNR) ||
 285             nvme_req(req)->retries >= nvme_max_retries)
 286                 return COMPLETE;
 287
 288         if (req->cmd_flags & REQ_NVME_MPATH) {
 289                 if (nvme_is_path_error(nvme_req(req)->status) ||
 290                     blk_queue_dying(req->q))
 291                         return FAILOVER;
 292         } else {
 293                 if (blk_queue_dying(req->q))
 294                         return COMPLETE;
 295         }
 296
 297         return RETRY;
 298 }
 299
 300 static inline void nvme_end_req(struct request *req)
 301 {
 302         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 303
 304         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 305             req_op(req) == REQ_OP_ZONE_APPEND)
 306                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 307                         le64_to_cpu(nvme_req(req)->result.u64));
 308
 309         nvme_trace_bio_complete(req, status);
 310         blk_mq_end_request(req, status);
 311 }
 312
 313 void nvme_complete_rq(struct request *req)
 314 {
 315         trace_nvme_complete_rq(req);
 316         nvme_cleanup_cmd(req);
 317
 318         if (nvme_req(req)->ctrl->kas)
 319                 nvme_req(req)->ctrl->comp_seen = true;
 320
 321         switch (nvme_decide_disposition(req)) {
 322         case COMPLETE:
 323                 nvme_end_req(req);
 324                 return;
 325         case RETRY:
 326                 nvme_retry_req(req);
 327                 return;
 328         case FAILOVER:
 329                 nvme_failover_req(req);
 330                 return;
 331         }
 332 }
 333 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 334
 335 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
 336 {
 337         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 338                                 "Cancelling I/O %d", req->tag);
 339
 340         /* don't abort one completed request */
 341         if (blk_mq_request_completed(req))
 342                 return true;
 343
 344         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 345         blk_mq_complete_request(req);
 346         return true;
 347 }
 348 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 349
 350 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 351                 enum nvme_ctrl_state new_state)
 352 {
 353         enum nvme_ctrl_state old_state;
 354         unsigned long flags;
 355         bool changed = false;
 356
 357         spin_lock_irqsave(&ctrl->lock, flags);
 358
 359         old_state = ctrl->state;
 360         switch (new_state) {
 361         case NVME_CTRL_LIVE:
 362                 switch (old_state) {
 363                 case NVME_CTRL_NEW:
 364                 case NVME_CTRL_RESETTING:
 365                 case NVME_CTRL_CONNECTING:
 366                         changed = true;
 367                         fallthrough;
 368                 default:
 369                         break;
 370                 }
 371                 break;
 372         case NVME_CTRL_RESETTING:
 373                 switch (old_state) {
 374                 case NVME_CTRL_NEW:
 375                 case NVME_CTRL_LIVE:
 376                         changed = true;
 377                         fallthrough;
 378                 default:
 379                         break;
 380                 }
 381                 break;
 382         case NVME_CTRL_CONNECTING:
 383                 switch (old_state) {
 384                 case NVME_CTRL_NEW:
 385                 case NVME_CTRL_RESETTING:
 386                         changed = true;
 387                         fallthrough;
 388                 default:
 389                         break;
 390                 }
 391                 break;
 392         case NVME_CTRL_DELETING:
 393                 switch (old_state) {
 394                 case NVME_CTRL_LIVE:
 395                 case NVME_CTRL_RESETTING:
 396                 case NVME_CTRL_CONNECTING:
 397                         changed = true;
 398                         fallthrough;
 399                 default:
 400                         break;
 401                 }
 402                 break;
 403         case NVME_CTRL_DELETING_NOIO:
 404                 switch (old_state) {
 405                 case NVME_CTRL_DELETING:
 406                 case NVME_CTRL_DEAD:
 407                         changed = true;
 408                         fallthrough;
 409                 default:
 410                         break;
 411                 }
 412                 break;
 413         case NVME_CTRL_DEAD:
 414                 switch (old_state) {
 415                 case NVME_CTRL_DELETING:
 416                         changed = true;
 417                         fallthrough;
 418                 default:
 419                         break;
 420                 }
 421                 break;
 422         default:
 423                 break;
 424         }
 425
 426         if (changed) {
 427                 ctrl->state = new_state;
 428                 wake_up_all(&ctrl->state_wq);
 429         }
 430
 431         spin_unlock_irqrestore(&ctrl->lock, flags);
 432         if (changed && ctrl->state == NVME_CTRL_LIVE)
 433                 nvme_kick_requeue_lists(ctrl);
 434         return changed;
 435 }
 436 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 437
 438 /*
 439  * Returns true for sink states that can't ever transition back to live.
 440  */
 441 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 442 {
 443         switch (ctrl->state) {
 444         case NVME_CTRL_NEW:
 445         case NVME_CTRL_LIVE:
 446         case NVME_CTRL_RESETTING:
 447         case NVME_CTRL_CONNECTING:
 448                 return false;
 449         case NVME_CTRL_DELETING:
 450         case NVME_CTRL_DELETING_NOIO:
 451         case NVME_CTRL_DEAD:
 452                 return true;
 453         default:
 454                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 455                 return true;
 456         }
 457 }
 458
 459 /*
 460  * Waits for the controller state to be resetting, or returns false if it is
 461  * not possible to ever transition to that state.
 462  */
 463 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 464 {
 465         wait_event(ctrl->state_wq,
 466                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 467                    nvme_state_terminal(ctrl));
 468         return ctrl->state == NVME_CTRL_RESETTING;
 469 }
 470 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 471
 472 static void nvme_free_ns_head(struct kref *ref)
 473 {
 474         struct nvme_ns_head *head =
 475                 container_of(ref, struct nvme_ns_head, ref);
 476
 477         nvme_mpath_remove_disk(head);
 478         ida_simple_remove(&head->subsys->ns_ida, head->instance);
 479         cleanup_srcu_struct(&head->srcu);
 480         nvme_put_subsystem(head->subsys);
 481         kfree(head);
 482 }
 483
 484 static void nvme_put_ns_head(struct nvme_ns_head *head)
 485 {
 486         kref_put(&head->ref, nvme_free_ns_head);
 487 }
 488
 489 static void nvme_free_ns(struct kref *kref)
 490 {
 491         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 492
 493         if (ns->ndev)
 494                 nvme_nvm_unregister(ns);
 495
 496         put_disk(ns->disk);
 497         nvme_put_ns_head(ns->head);
 498         nvme_put_ctrl(ns->ctrl);
 499         kfree(ns);
 500 }
 501
 502 void nvme_put_ns(struct nvme_ns *ns)
 503 {
 504         kref_put(&ns->kref, nvme_free_ns);
 505 }
 506 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 507
 508 static inline void nvme_clear_nvme_request(struct request *req)
 509 {
 510         if (!(req->rq_flags & RQF_DONTPREP)) {
 511                 nvme_req(req)->retries = 0;
 512                 nvme_req(req)->flags = 0;
 513                 req->rq_flags |= RQF_DONTPREP;
 514         }
 515 }
 516
 517 struct request *nvme_alloc_request(struct request_queue *q,
 518                 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
 519 {
 520         unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
 521         struct request *req;
 522
 523         if (qid == NVME_QID_ANY) {
 524                 req = blk_mq_alloc_request(q, op, flags);
 525         } else {
 526                 req = blk_mq_alloc_request_hctx(q, op, flags,
 527                                 qid ? qid - 1 : 0);
 528         }
 529         if (IS_ERR(req))
 530                 return req;
 531
 532         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 533         nvme_clear_nvme_request(req);
 534         nvme_req(req)->cmd = cmd;
 535
 536         return req;
 537 }
 538 EXPORT_SYMBOL_GPL(nvme_alloc_request);
 539
 540 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
 541 {
 542         struct nvme_command c;
 543
 544         memset(&c, 0, sizeof(c));
 545
 546         c.directive.opcode = nvme_admin_directive_send;
 547         c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
 548         c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
 549         c.directive.dtype = NVME_DIR_IDENTIFY;
 550         c.directive.tdtype = NVME_DIR_STREAMS;
 551         c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
 552
 553         return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
 554 }
 555
 556 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
 557 {
 558         return nvme_toggle_streams(ctrl, false);
 559 }
 560
 561 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
 562 {
 563         return nvme_toggle_streams(ctrl, true);
 564 }
 565
 566 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
 567                                   struct streams_directive_params *s, u32 nsid)
 568 {
 569         struct nvme_command c;
 570
 571         memset(&c, 0, sizeof(c));
 572         memset(s, 0, sizeof(*s));
 573
 574         c.directive.opcode = nvme_admin_directive_recv;
 575         c.directive.nsid = cpu_to_le32(nsid);
 576         c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
 577         c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
 578         c.directive.dtype = NVME_DIR_STREAMS;
 579
 580         return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
 581 }
 582
 583 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
 584 {
 585         struct streams_directive_params s;
 586         int ret;
 587
 588         if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
 589                 return 0;
 590         if (!streams)
 591                 return 0;
 592
 593         ret = nvme_enable_streams(ctrl);
 594         if (ret)
 595                 return ret;
 596
 597         ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
 598         if (ret)
 599                 goto out_disable_stream;
 600
 601         ctrl->nssa = le16_to_cpu(s.nssa);
 602         if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
 603                 dev_info(ctrl->device, "too few streams (%u) available\n",
 604                                         ctrl->nssa);
 605                 goto out_disable_stream;
 606         }
 607
 608         ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
 609         dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
 610         return 0;
 611
 612 out_disable_stream:
 613         nvme_disable_streams(ctrl);
 614         return ret;
 615 }
 616
 617 /*
 618  * Check if 'req' has a write hint associated with it. If it does, assign
 619  * a valid namespace stream to the write.
 620  */
 621 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
 622                                      struct request *req, u16 *control,
 623                                      u32 *dsmgmt)
 624 {
 625         enum rw_hint streamid = req->write_hint;
 626
 627         if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
 628                 streamid = 0;
 629         else {
 630                 streamid--;
 631                 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
 632                         return;
 633
 634                 *control |= NVME_RW_DTYPE_STREAMS;
 635                 *dsmgmt |= streamid << 16;
 636         }
 637
 638         if (streamid < ARRAY_SIZE(req->q->write_hints))
 639                 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
 640 }
 641
 642 static void nvme_setup_passthrough(struct request *req,
 643                 struct nvme_command *cmd)
 644 {
 645         memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
 646         /* passthru commands should let the driver set the SGL flags */
 647         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 648 }
 649
 650 static inline void nvme_setup_flush(struct nvme_ns *ns,
 651                 struct nvme_command *cmnd)
 652 {
 653         cmnd->common.opcode = nvme_cmd_flush;
 654         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 655 }
 656
 657 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 658                 struct nvme_command *cmnd)
 659 {
 660         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 661         struct nvme_dsm_range *range;
 662         struct bio *bio;
 663
 664         /*
 665          * Some devices do not consider the DSM 'Number of Ranges' field when
 666          * determining how much data to DMA. Always allocate memory for maximum
 667          * number of segments to prevent device reading beyond end of buffer.
 668          */
 669         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 670
 671         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 672         if (!range) {
 673                 /*
 674                  * If we fail allocation our range, fallback to the controller
 675                  * discard page. If that's also busy, it's safe to return
 676                  * busy, as we know we can make progress once that's freed.
 677                  */
 678                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 679                         return BLK_STS_RESOURCE;
 680
 681                 range = page_address(ns->ctrl->discard_page);
 682         }
 683
 684         __rq_for_each_bio(bio, req) {
 685                 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 686                 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 687
 688                 if (n < segments) {
 689                         range[n].cattr = cpu_to_le32(0);
 690                         range[n].nlb = cpu_to_le32(nlb);
 691                         range[n].slba = cpu_to_le64(slba);
 692                 }
 693                 n++;
 694         }
 695
 696         if (WARN_ON_ONCE(n != segments)) {
 697                 if (virt_to_page(range) == ns->ctrl->discard_page)
 698                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 699                 else
 700                         kfree(range);
 701                 return BLK_STS_IOERR;
 702         }
 703
 704         cmnd->dsm.opcode = nvme_cmd_dsm;
 705         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 706         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 707         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 708
 709         req->special_vec.bv_page = virt_to_page(range);
 710         req->special_vec.bv_offset = offset_in_page(range);
 711         req->special_vec.bv_len = alloc_size;
 712         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 713
 714         return BLK_STS_OK;
 715 }
 716
 717 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 718                 struct request *req, struct nvme_command *cmnd)
 719 {
 720         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 721                 return nvme_setup_discard(ns, req, cmnd);
 722
 723         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 724         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 725         cmnd->write_zeroes.slba =
 726                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 727         cmnd->write_zeroes.length =
 728                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 729         cmnd->write_zeroes.control = 0;
 730         return BLK_STS_OK;
 731 }
 732
 733 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 734                 struct request *req, struct nvme_command *cmnd,
 735                 enum nvme_opcode op)
 736 {
 737         struct nvme_ctrl *ctrl = ns->ctrl;
 738         u16 control = 0;
 739         u32 dsmgmt = 0;
 740
 741         if (req->cmd_flags & REQ_FUA)
 742                 control |= NVME_RW_FUA;
 743         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 744                 control |= NVME_RW_LR;
 745
 746         if (req->cmd_flags & REQ_RAHEAD)
 747                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 748
 749         cmnd->rw.opcode = op;
 750         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 751         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 752         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 753
 754         if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
 755                 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
 756
 757         if (ns->ms) {
 758                 /*
 759                  * If formated with metadata, the block layer always provides a
 760                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 761                  * we enable the PRACT bit for protection information or set the
 762                  * namespace capacity to zero to prevent any I/O.
 763                  */
 764                 if (!blk_integrity_rq(req)) {
 765                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 766                                 return BLK_STS_NOTSUPP;
 767                         control |= NVME_RW_PRINFO_PRACT;
 768                 }
 769
 770                 switch (ns->pi_type) {
 771                 case NVME_NS_DPS_PI_TYPE3:
 772                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 773                         break;
 774                 case NVME_NS_DPS_PI_TYPE1:
 775                 case NVME_NS_DPS_PI_TYPE2:
 776                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 777                                         NVME_RW_PRINFO_PRCHK_REF;
 778                         if (op == nvme_cmd_zone_append)
 779                                 control |= NVME_RW_APPEND_PIREMAP;
 780                         cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 781                         break;
 782                 }
 783         }
 784
 785         cmnd->rw.control = cpu_to_le16(control);
 786         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 787         return 0;
 788 }
 789
 790 void nvme_cleanup_cmd(struct request *req)
 791 {
 792         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 793                 struct nvme_ns *ns = req->rq_disk->private_data;
 794                 struct page *page = req->special_vec.bv_page;
 795
 796                 if (page == ns->ctrl->discard_page)
 797                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 798                 else
 799                         kfree(page_address(page) + req->special_vec.bv_offset);
 800         }
 801 }
 802 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 803
 804 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 805                 struct nvme_command *cmd)
 806 {
 807         blk_status_t ret = BLK_STS_OK;
 808
 809         nvme_clear_nvme_request(req);
 810
 811         memset(cmd, 0, sizeof(*cmd));
 812         switch (req_op(req)) {
 813         case REQ_OP_DRV_IN:
 814         case REQ_OP_DRV_OUT:
 815                 nvme_setup_passthrough(req, cmd);
 816                 break;
 817         case REQ_OP_FLUSH:
 818                 nvme_setup_flush(ns, cmd);
 819                 break;
 820         case REQ_OP_ZONE_RESET_ALL:
 821         case REQ_OP_ZONE_RESET:
 822                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 823                 break;
 824         case REQ_OP_ZONE_OPEN:
 825                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 826                 break;
 827         case REQ_OP_ZONE_CLOSE:
 828                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 829                 break;
 830         case REQ_OP_ZONE_FINISH:
 831                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 832                 break;
 833         case REQ_OP_WRITE_ZEROES:
 834                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 835                 break;
 836         case REQ_OP_DISCARD:
 837                 ret = nvme_setup_discard(ns, req, cmd);
 838                 break;
 839         case REQ_OP_READ:
 840                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 841                 break;
 842         case REQ_OP_WRITE:
 843                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 844                 break;
 845         case REQ_OP_ZONE_APPEND:
 846                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 847                 break;
 848         default:
 849                 WARN_ON_ONCE(1);
 850                 return BLK_STS_IOERR;
 851         }
 852
 853         cmd->common.command_id = req->tag;
 854         trace_nvme_setup_cmd(req, cmd);
 855         return ret;
 856 }
 857 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
 858
 859 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
 860 {
 861         struct completion *waiting = rq->end_io_data;
 862
 863         rq->end_io_data = NULL;
 864         complete(waiting);
 865 }
 866
 867 static void nvme_execute_rq_polled(struct request_queue *q,
 868                 struct gendisk *bd_disk, struct request *rq, int at_head)
 869 {
 870         DECLARE_COMPLETION_ONSTACK(wait);
 871
 872         WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
 873
 874         rq->cmd_flags |= REQ_HIPRI;
 875         rq->end_io_data = &wait;
 876         blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
 877
 878         while (!completion_done(&wait)) {
 879                 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
 880                 cond_resched();
 881         }
 882 }
 883
 884 /*
 885  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 886  * if the result is positive, it's an NVM Express status code
 887  */
 888 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 889                 union nvme_result *result, void *buffer, unsigned bufflen,
 890                 unsigned timeout, int qid, int at_head,
 891                 blk_mq_req_flags_t flags, bool poll)
 892 {
 893         struct request *req;
 894         int ret;
 895
 896         req = nvme_alloc_request(q, cmd, flags, qid);
 897         if (IS_ERR(req))
 898                 return PTR_ERR(req);
 899
 900         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
 901
 902         if (buffer && bufflen) {
 903                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
 904                 if (ret)
 905                         goto out;
 906         }
 907
 908         if (poll)
 909                 nvme_execute_rq_polled(req->q, NULL, req, at_head);
 910         else
 911                 blk_execute_rq(req->q, NULL, req, at_head);
 912         if (result)
 913                 *result = nvme_req(req)->result;
 914         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 915                 ret = -EINTR;
 916         else
 917                 ret = nvme_req(req)->status;
 918  out:
 919         blk_mq_free_request(req);
 920         return ret;
 921 }
 922 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
 923
 924 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 925                 void *buffer, unsigned bufflen)
 926 {
 927         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
 928                         NVME_QID_ANY, 0, 0, false);
 929 }
 930 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 931
 932 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
 933                 unsigned len, u32 seed, bool write)
 934 {
 935         struct bio_integrity_payload *bip;
 936         int ret = -ENOMEM;
 937         void *buf;
 938
 939         buf = kmalloc(len, GFP_KERNEL);
 940         if (!buf)
 941                 goto out;
 942
 943         ret = -EFAULT;
 944         if (write && copy_from_user(buf, ubuf, len))
 945                 goto out_free_meta;
 946
 947         bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
 948         if (IS_ERR(bip)) {
 949                 ret = PTR_ERR(bip);
 950                 goto out_free_meta;
 951         }
 952
 953         bip->bip_iter.bi_size = len;
 954         bip->bip_iter.bi_sector = seed;
 955         ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
 956                         offset_in_page(buf));
 957         if (ret == len)
 958                 return buf;
 959         ret = -ENOMEM;
 960 out_free_meta:
 961         kfree(buf);
 962 out:
 963         return ERR_PTR(ret);
 964 }
 965
 966 static u32 nvme_known_admin_effects(u8 opcode)
 967 {
 968         switch (opcode) {
 969         case nvme_admin_format_nvm:
 970                 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC |
 971                         NVME_CMD_EFFECTS_CSE_MASK;
 972         case nvme_admin_sanitize_nvm:
 973                 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK;
 974         default:
 975                 break;
 976         }
 977         return 0;
 978 }
 979
 980 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
 981 {
 982         u32 effects = 0;
 983
 984         if (ns) {
 985                 if (ns->head->effects)
 986                         effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
 987                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
 988                         dev_warn(ctrl->device,
 989                                  "IO command:%02x has unhandled effects:%08x\n",
 990                                  opcode, effects);
 991                 return 0;
 992         }
 993
 994         if (ctrl->effects)
 995                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
 996         effects |= nvme_known_admin_effects(opcode);
 997
 998         return effects;
 999 }
1000 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1001
1002 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1003                                u8 opcode)
1004 {
1005         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1006
1007         /*
1008          * For simplicity, IO to all namespaces is quiesced even if the command
1009          * effects say only one namespace is affected.
1010          */
1011         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1012                 mutex_lock(&ctrl->scan_lock);
1013                 mutex_lock(&ctrl->subsys->lock);
1014                 nvme_mpath_start_freeze(ctrl->subsys);
1015                 nvme_mpath_wait_freeze(ctrl->subsys);
1016                 nvme_start_freeze(ctrl);
1017                 nvme_wait_freeze(ctrl);
1018         }
1019         return effects;
1020 }
1021
1022 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1023 {
1024         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1025                 nvme_unfreeze(ctrl);
1026                 nvme_mpath_unfreeze(ctrl->subsys);
1027                 mutex_unlock(&ctrl->subsys->lock);
1028                 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1029                 mutex_unlock(&ctrl->scan_lock);
1030         }
1031         if (effects & NVME_CMD_EFFECTS_CCC)
1032                 nvme_init_identify(ctrl);
1033         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1034                 nvme_queue_scan(ctrl);
1035                 flush_work(&ctrl->scan_work);
1036         }
1037 }
1038
1039 void nvme_execute_passthru_rq(struct request *rq)
1040 {
1041         struct nvme_command *cmd = nvme_req(rq)->cmd;
1042         struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1043         struct nvme_ns *ns = rq->q->queuedata;
1044         struct gendisk *disk = ns ? ns->disk : NULL;
1045         u32 effects;
1046
1047         effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1048         blk_execute_rq(rq->q, disk, rq, 0);
1049         nvme_passthru_end(ctrl, effects);
1050 }
1051 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1052
1053 static int nvme_submit_user_cmd(struct request_queue *q,
1054                 struct nvme_command *cmd, void __user *ubuffer,
1055                 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
1056                 u32 meta_seed, u64 *result, unsigned timeout)
1057 {
1058         bool write = nvme_is_write(cmd);
1059         struct nvme_ns *ns = q->queuedata;
1060         struct gendisk *disk = ns ? ns->disk : NULL;
1061         struct request *req;
1062         struct bio *bio = NULL;
1063         void *meta = NULL;
1064         int ret;
1065
1066         req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
1067         if (IS_ERR(req))
1068                 return PTR_ERR(req);
1069
1070         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
1071         nvme_req(req)->flags |= NVME_REQ_USERCMD;
1072
1073         if (ubuffer && bufflen) {
1074                 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
1075                                 GFP_KERNEL);
1076                 if (ret)
1077                         goto out;
1078                 bio = req->bio;
1079                 bio->bi_disk = disk;
1080                 if (disk && meta_buffer && meta_len) {
1081                         meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
1082                                         meta_seed, write);
1083                         if (IS_ERR(meta)) {
1084                                 ret = PTR_ERR(meta);
1085                                 goto out_unmap;
1086                         }
1087                         req->cmd_flags |= REQ_INTEGRITY;
1088                 }
1089         }
1090
1091         nvme_execute_passthru_rq(req);
1092         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
1093                 ret = -EINTR;
1094         else
1095                 ret = nvme_req(req)->status;
1096         if (result)
1097                 *result = le64_to_cpu(nvme_req(req)->result.u64);
1098         if (meta && !ret && !write) {
1099                 if (copy_to_user(meta_buffer, meta, meta_len))
1100                         ret = -EFAULT;
1101         }
1102         kfree(meta);
1103  out_unmap:
1104         if (bio)
1105                 blk_rq_unmap_user(bio);
1106  out:
1107         blk_mq_free_request(req);
1108         return ret;
1109 }
1110
1111 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
1112 {
1113         struct nvme_ctrl *ctrl = rq->end_io_data;
1114         unsigned long flags;
1115         bool startka = false;
1116
1117         blk_mq_free_request(rq);
1118
1119         if (status) {
1120                 dev_err(ctrl->device,
1121                         "failed nvme_keep_alive_end_io error=%d\n",
1122                                 status);
1123                 return;
1124         }
1125
1126         ctrl->comp_seen = false;
1127         spin_lock_irqsave(&ctrl->lock, flags);
1128         if (ctrl->state == NVME_CTRL_LIVE ||
1129             ctrl->state == NVME_CTRL_CONNECTING)
1130                 startka = true;
1131         spin_unlock_irqrestore(&ctrl->lock, flags);
1132         if (startka)
1133                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1134 }
1135
1136 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
1137 {
1138         struct request *rq;
1139
1140         rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
1141                         NVME_QID_ANY);
1142         if (IS_ERR(rq))
1143                 return PTR_ERR(rq);
1144
1145         rq->timeout = ctrl->kato * HZ;
1146         rq->end_io_data = ctrl;
1147
1148         blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1149
1150         return 0;
1151 }
1152
1153 static void nvme_keep_alive_work(struct work_struct *work)
1154 {
1155         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1156                         struct nvme_ctrl, ka_work);
1157         bool comp_seen = ctrl->comp_seen;
1158
1159         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1160                 dev_dbg(ctrl->device,
1161                         "reschedule traffic based keep-alive timer\n");
1162                 ctrl->comp_seen = false;
1163                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1164                 return;
1165         }
1166
1167         if (nvme_keep_alive(ctrl)) {
1168                 /* allocation failure, reset the controller */
1169                 dev_err(ctrl->device, "keep-alive failed\n");
1170                 nvme_reset_ctrl(ctrl);
1171                 return;
1172         }
1173 }
1174
1175 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1176 {
1177         if (unlikely(ctrl->kato == 0))
1178                 return;
1179
1180         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1181 }
1182
1183 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1184 {
1185         if (unlikely(ctrl->kato == 0))
1186                 return;
1187
1188         cancel_delayed_work_sync(&ctrl->ka_work);
1189 }
1190 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1191
1192 /*
1193  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1194  * flag, thus sending any new CNS opcodes has a big chance of not working.
1195  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1196  * (but not for any later version).
1197  */
1198 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1199 {
1200         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1201                 return ctrl->vs < NVME_VS(1, 2, 0);
1202         return ctrl->vs < NVME_VS(1, 1, 0);
1203 }
1204
1205 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1206 {
1207         struct nvme_command c = { };
1208         int error;
1209
1210         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1211         c.identify.opcode = nvme_admin_identify;
1212         c.identify.cns = NVME_ID_CNS_CTRL;
1213
1214         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1215         if (!*id)
1216                 return -ENOMEM;
1217
1218         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1219                         sizeof(struct nvme_id_ctrl));
1220         if (error)
1221                 kfree(*id);
1222         return error;
1223 }
1224
1225 static bool nvme_multi_css(struct nvme_ctrl *ctrl)
1226 {
1227         return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
1228 }
1229
1230 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1231                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1232 {
1233         const char *warn_str = "ctrl returned bogus length:";
1234         void *data = cur;
1235
1236         switch (cur->nidt) {
1237         case NVME_NIDT_EUI64:
1238                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1239                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1240                                  warn_str, cur->nidl);
1241                         return -1;
1242                 }
1243                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1244                 return NVME_NIDT_EUI64_LEN;
1245         case NVME_NIDT_NGUID:
1246                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1247                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1248                                  warn_str, cur->nidl);
1249                         return -1;
1250                 }
1251                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1252                 return NVME_NIDT_NGUID_LEN;
1253         case NVME_NIDT_UUID:
1254                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1255                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1256                                  warn_str, cur->nidl);
1257                         return -1;
1258                 }
1259                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1260                 return NVME_NIDT_UUID_LEN;
1261         case NVME_NIDT_CSI:
1262                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1263                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1264                                  warn_str, cur->nidl);
1265                         return -1;
1266                 }
1267                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1268                 *csi_seen = true;
1269                 return NVME_NIDT_CSI_LEN;
1270         default:
1271                 /* Skip unknown types */
1272                 return cur->nidl;
1273         }
1274 }
1275
1276 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1277                 struct nvme_ns_ids *ids)
1278 {
1279         struct nvme_command c = { };
1280         bool csi_seen = false;
1281         int status, pos, len;
1282         void *data;
1283
1284         if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1285                 return 0;
1286         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1287                 return 0;
1288
1289         c.identify.opcode = nvme_admin_identify;
1290         c.identify.nsid = cpu_to_le32(nsid);
1291         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1292
1293         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1294         if (!data)
1295                 return -ENOMEM;
1296
1297         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1298                                       NVME_IDENTIFY_DATA_SIZE);
1299         if (status) {
1300                 dev_warn(ctrl->device,
1301                         "Identify Descriptors failed (%d)\n", status);
1302                 goto free_data;
1303         }
1304
1305         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1306                 struct nvme_ns_id_desc *cur = data + pos;
1307
1308                 if (cur->nidl == 0)
1309                         break;
1310
1311                 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
1312                 if (len < 0)
1313                         break;
1314
1315                 len += sizeof(*cur);
1316         }
1317
1318         if (nvme_multi_css(ctrl) && !csi_seen) {
1319                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1320                          nsid);
1321                 status = -EINVAL;
1322         }
1323
1324 free_data:
1325         kfree(data);
1326         return status;
1327 }
1328
1329 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1330                         struct nvme_ns_ids *ids, struct nvme_id_ns **id)
1331 {
1332         struct nvme_command c = { };
1333         int error;
1334
1335         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1336         c.identify.opcode = nvme_admin_identify;
1337         c.identify.nsid = cpu_to_le32(nsid);
1338         c.identify.cns = NVME_ID_CNS_NS;
1339
1340         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1341         if (!*id)
1342                 return -ENOMEM;
1343
1344         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1345         if (error) {
1346                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1347                 goto out_free_id;
1348         }
1349
1350         error = -ENODEV;
1351         if ((*id)->ncap == 0) /* namespace not allocated or attached */
1352                 goto out_free_id;
1353
1354         if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1355             !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1356                 memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
1357         if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1358             !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1359                 memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
1360
1361         return 0;
1362
1363 out_free_id:
1364         kfree(*id);
1365         return error;
1366 }
1367
1368 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1369                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1370 {
1371         union nvme_result res = { 0 };
1372         struct nvme_command c;
1373         int ret;
1374
1375         memset(&c, 0, sizeof(c));
1376         c.features.opcode = op;
1377         c.features.fid = cpu_to_le32(fid);
1378         c.features.dword11 = cpu_to_le32(dword11);
1379
1380         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1381                         buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1382         if (ret >= 0 && result)
1383                 *result = le32_to_cpu(res.u32);
1384         return ret;
1385 }
1386
1387 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1388                       unsigned int dword11, void *buffer, size_t buflen,
1389                       u32 *result)
1390 {
1391         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1392                              buflen, result);
1393 }
1394 EXPORT_SYMBOL_GPL(nvme_set_features);
1395
1396 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1397                       unsigned int dword11, void *buffer, size_t buflen,
1398                       u32 *result)
1399 {
1400         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1401                              buflen, result);
1402 }
1403 EXPORT_SYMBOL_GPL(nvme_get_features);
1404
1405 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1406 {
1407         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1408         u32 result;
1409         int status, nr_io_queues;
1410
1411         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1412                         &result);
1413         if (status < 0)
1414                 return status;
1415
1416         /*
1417          * Degraded controllers might return an error when setting the queue
1418          * count.  We still want to be able to bring them online and offer
1419          * access to the admin queue, as that might be only way to fix them up.
1420          */
1421         if (status > 0) {
1422                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1423                 *count = 0;
1424         } else {
1425                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1426                 *count = min(*count, nr_io_queues);
1427         }
1428
1429         return 0;
1430 }
1431 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1432
1433 #define NVME_AEN_SUPPORTED \
1434         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1435          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1436
1437 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1438 {
1439         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1440         int status;
1441
1442         if (!supported_aens)
1443                 return;
1444
1445         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1446                         NULL, 0, &result);
1447         if (status)
1448                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1449                          supported_aens);
1450
1451         queue_work(nvme_wq, &ctrl->async_event_work);
1452 }
1453
1454 /*
1455  * Convert integer values from ioctl structures to user pointers, silently
1456  * ignoring the upper bits in the compat case to match behaviour of 32-bit
1457  * kernels.
1458  */
1459 static void __user *nvme_to_user_ptr(uintptr_t ptrval)
1460 {
1461         if (in_compat_syscall())
1462                 ptrval = (compat_uptr_t)ptrval;
1463         return (void __user *)ptrval;
1464 }
1465
1466 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1467 {
1468         struct nvme_user_io io;
1469         struct nvme_command c;
1470         unsigned length, meta_len;
1471         void __user *metadata;
1472
1473         if (copy_from_user(&io, uio, sizeof(io)))
1474                 return -EFAULT;
1475         if (io.flags)
1476                 return -EINVAL;
1477
1478         switch (io.opcode) {
1479         case nvme_cmd_write:
1480         case nvme_cmd_read:
1481         case nvme_cmd_compare:
1482                 break;
1483         default:
1484                 return -EINVAL;
1485         }
1486
1487         length = (io.nblocks + 1) << ns->lba_shift;
1488         meta_len = (io.nblocks + 1) * ns->ms;
1489         metadata = nvme_to_user_ptr(io.metadata);
1490
1491         if (ns->features & NVME_NS_EXT_LBAS) {
1492                 length += meta_len;
1493                 meta_len = 0;
1494         } else if (meta_len) {
1495                 if ((io.metadata & 3) || !io.metadata)
1496                         return -EINVAL;
1497         }
1498
1499         memset(&c, 0, sizeof(c));
1500         c.rw.opcode = io.opcode;
1501         c.rw.flags = io.flags;
1502         c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1503         c.rw.slba = cpu_to_le64(io.slba);
1504         c.rw.length = cpu_to_le16(io.nblocks);
1505         c.rw.control = cpu_to_le16(io.control);
1506         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1507         c.rw.reftag = cpu_to_le32(io.reftag);
1508         c.rw.apptag = cpu_to_le16(io.apptag);
1509         c.rw.appmask = cpu_to_le16(io.appmask);
1510
1511         return nvme_submit_user_cmd(ns->queue, &c,
1512                         nvme_to_user_ptr(io.addr), length,
1513                         metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1514 }
1515
1516 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1517                         struct nvme_passthru_cmd __user *ucmd)
1518 {
1519         struct nvme_passthru_cmd cmd;
1520         struct nvme_command c;
1521         unsigned timeout = 0;
1522         u64 result;
1523         int status;
1524
1525         if (!capable(CAP_SYS_ADMIN))
1526                 return -EACCES;
1527         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1528                 return -EFAULT;
1529         if (cmd.flags)
1530                 return -EINVAL;
1531
1532         memset(&c, 0, sizeof(c));
1533         c.common.opcode = cmd.opcode;
1534         c.common.flags = cmd.flags;
1535         c.common.nsid = cpu_to_le32(cmd.nsid);
1536         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1537         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1538         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1539         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1540         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1541         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1542         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1543         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1544
1545         if (cmd.timeout_ms)
1546                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1547
1548         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1549                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1550                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1551                         0, &result, timeout);
1552
1553         if (status >= 0) {
1554                 if (put_user(result, &ucmd->result))
1555                         return -EFAULT;
1556         }
1557
1558         return status;
1559 }
1560
1561 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1562                         struct nvme_passthru_cmd64 __user *ucmd)
1563 {
1564         struct nvme_passthru_cmd64 cmd;
1565         struct nvme_command c;
1566         unsigned timeout = 0;
1567         int status;
1568
1569         if (!capable(CAP_SYS_ADMIN))
1570                 return -EACCES;
1571         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1572                 return -EFAULT;
1573         if (cmd.flags)
1574                 return -EINVAL;
1575
1576         memset(&c, 0, sizeof(c));
1577         c.common.opcode = cmd.opcode;
1578         c.common.flags = cmd.flags;
1579         c.common.nsid = cpu_to_le32(cmd.nsid);
1580         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1581         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1582         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1583         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1584         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1585         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1586         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1587         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1588
1589         if (cmd.timeout_ms)
1590                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1591
1592         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1593                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1594                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1595                         0, &cmd.result, timeout);
1596
1597         if (status >= 0) {
1598                 if (put_user(cmd.result, &ucmd->result))
1599                         return -EFAULT;
1600         }
1601
1602         return status;
1603 }
1604
1605 /*
1606  * Issue ioctl requests on the first available path.  Note that unlike normal
1607  * block layer requests we will not retry failed request on another controller.
1608  */
1609 struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1610                 struct nvme_ns_head **head, int *srcu_idx)
1611 {
1612 #ifdef CONFIG_NVME_MULTIPATH
1613         if (disk->fops == &nvme_ns_head_ops) {
1614                 struct nvme_ns *ns;
1615
1616                 *head = disk->private_data;
1617                 *srcu_idx = srcu_read_lock(&(*head)->srcu);
1618                 ns = nvme_find_path(*head);
1619                 if (!ns)
1620                         srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1621                 return ns;
1622         }
1623 #endif
1624         *head = NULL;
1625         *srcu_idx = -1;
1626         return disk->private_data;
1627 }
1628
1629 void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1630 {
1631         if (head)
1632                 srcu_read_unlock(&head->srcu, idx);
1633 }
1634
1635 static bool is_ctrl_ioctl(unsigned int cmd)
1636 {
1637         if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1638                 return true;
1639         if (is_sed_ioctl(cmd))
1640                 return true;
1641         return false;
1642 }
1643
1644 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1645                                   void __user *argp,
1646                                   struct nvme_ns_head *head,
1647                                   int srcu_idx)
1648 {
1649         struct nvme_ctrl *ctrl = ns->ctrl;
1650         int ret;
1651
1652         nvme_get_ctrl(ns->ctrl);
1653         nvme_put_ns_from_disk(head, srcu_idx);
1654
1655         switch (cmd) {
1656         case NVME_IOCTL_ADMIN_CMD:
1657                 ret = nvme_user_cmd(ctrl, NULL, argp);
1658                 break;
1659         case NVME_IOCTL_ADMIN64_CMD:
1660                 ret = nvme_user_cmd64(ctrl, NULL, argp);
1661                 break;
1662         default:
1663                 ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1664                 break;
1665         }
1666         nvme_put_ctrl(ctrl);
1667         return ret;
1668 }
1669
1670 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1671                 unsigned int cmd, unsigned long arg)
1672 {
1673         struct nvme_ns_head *head = NULL;
1674         void __user *argp = (void __user *)arg;
1675         struct nvme_ns *ns;
1676         int srcu_idx, ret;
1677
1678         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1679         if (unlikely(!ns))
1680                 return -EWOULDBLOCK;
1681
1682         /*
1683          * Handle ioctls that apply to the controller instead of the namespace
1684          * seperately and drop the ns SRCU reference early.  This avoids a
1685          * deadlock when deleting namespaces using the passthrough interface.
1686          */
1687         if (is_ctrl_ioctl(cmd))
1688                 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1689
1690         switch (cmd) {
1691         case NVME_IOCTL_ID:
1692                 force_successful_syscall_return();
1693                 ret = ns->head->ns_id;
1694                 break;
1695         case NVME_IOCTL_IO_CMD:
1696                 ret = nvme_user_cmd(ns->ctrl, ns, argp);
1697                 break;
1698         case NVME_IOCTL_SUBMIT_IO:
1699                 ret = nvme_submit_io(ns, argp);
1700                 break;
1701         case NVME_IOCTL_IO64_CMD:
1702                 ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1703                 break;
1704         default:
1705                 if (ns->ndev)
1706                         ret = nvme_nvm_ioctl(ns, cmd, arg);
1707                 else
1708                         ret = -ENOTTY;
1709         }
1710
1711         nvme_put_ns_from_disk(head, srcu_idx);
1712         return ret;
1713 }
1714
1715 #ifdef CONFIG_COMPAT
1716 struct nvme_user_io32 {
1717         __u8    opcode;
1718         __u8    flags;
1719         __u16   control;
1720         __u16   nblocks;
1721         __u16   rsvd;
1722         __u64   metadata;
1723         __u64   addr;
1724         __u64   slba;
1725         __u32   dsmgmt;
1726         __u32   reftag;
1727         __u16   apptag;
1728         __u16   appmask;
1729 } __attribute__((__packed__));
1730
1731 #define NVME_IOCTL_SUBMIT_IO32  _IOW('N', 0x42, struct nvme_user_io32)
1732
1733 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1734                 unsigned int cmd, unsigned long arg)
1735 {
1736         /*
1737          * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
1738          * between 32 bit programs and 64 bit kernel.
1739          * The cause is that the results of sizeof(struct nvme_user_io),
1740          * which is used to define NVME_IOCTL_SUBMIT_IO,
1741          * are not same between 32 bit compiler and 64 bit compiler.
1742          * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
1743          * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
1744          * Other IOCTL numbers are same between 32 bit and 64 bit.
1745          * So there is nothing to do regarding to other IOCTL numbers.
1746          */
1747         if (cmd == NVME_IOCTL_SUBMIT_IO32)
1748                 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
1749
1750         return nvme_ioctl(bdev, mode, cmd, arg);
1751 }
1752 #else
1753 #define nvme_compat_ioctl       NULL
1754 #endif /* CONFIG_COMPAT */
1755
1756 static int nvme_open(struct block_device *bdev, fmode_t mode)
1757 {
1758         struct nvme_ns *ns = bdev->bd_disk->private_data;
1759
1760 #ifdef CONFIG_NVME_MULTIPATH
1761         /* should never be called due to GENHD_FL_HIDDEN */
1762         if (WARN_ON_ONCE(ns->head->disk))
1763                 goto fail;
1764 #endif
1765         if (!kref_get_unless_zero(&ns->kref))
1766                 goto fail;
1767         if (!try_module_get(ns->ctrl->ops->module))
1768                 goto fail_put_ns;
1769
1770         return 0;
1771
1772 fail_put_ns:
1773         nvme_put_ns(ns);
1774 fail:
1775         return -ENXIO;
1776 }
1777
1778 static void nvme_release(struct gendisk *disk, fmode_t mode)
1779 {
1780         struct nvme_ns *ns = disk->private_data;
1781
1782         module_put(ns->ctrl->ops->module);
1783         nvme_put_ns(ns);
1784 }
1785
1786 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1787 {
1788         /* some standard values */
1789         geo->heads = 1 << 6;
1790         geo->sectors = 1 << 5;
1791         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1792         return 0;
1793 }
1794
1795 #ifdef CONFIG_BLK_DEV_INTEGRITY
1796 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1797                                 u32 max_integrity_segments)
1798 {
1799         struct blk_integrity integrity;
1800
1801         memset(&integrity, 0, sizeof(integrity));
1802         switch (pi_type) {
1803         case NVME_NS_DPS_PI_TYPE3:
1804                 integrity.profile = &t10_pi_type3_crc;
1805                 integrity.tag_size = sizeof(u16) + sizeof(u32);
1806                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1807                 break;
1808         case NVME_NS_DPS_PI_TYPE1:
1809         case NVME_NS_DPS_PI_TYPE2:
1810                 integrity.profile = &t10_pi_type1_crc;
1811                 integrity.tag_size = sizeof(u16);
1812                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1813                 break;
1814         default:
1815                 integrity.profile = NULL;
1816                 break;
1817         }
1818         integrity.tuple_size = ms;
1819         blk_integrity_register(disk, &integrity);
1820         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1821 }
1822 #else
1823 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1824                                 u32 max_integrity_segments)
1825 {
1826 }
1827 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1828
1829 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1830 {
1831         struct nvme_ctrl *ctrl = ns->ctrl;
1832         struct request_queue *queue = disk->queue;
1833         u32 size = queue_logical_block_size(queue);
1834
1835         if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1836                 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1837                 return;
1838         }
1839
1840         if (ctrl->nr_streams && ns->sws && ns->sgs)
1841                 size *= ns->sws * ns->sgs;
1842
1843         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1844                         NVME_DSM_MAX_RANGES);
1845
1846         queue->limits.discard_alignment = 0;
1847         queue->limits.discard_granularity = size;
1848
1849         /* If discard is already enabled, don't reset queue limits */
1850         if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1851                 return;
1852
1853         blk_queue_max_discard_sectors(queue, UINT_MAX);
1854         blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1855
1856         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1857                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1858 }
1859
1860 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
1861 {
1862         u64 max_blocks;
1863
1864         if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1865             (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1866                 return;
1867         /*
1868          * Even though NVMe spec explicitly states that MDTS is not
1869          * applicable to the write-zeroes:- "The restriction does not apply to
1870          * commands that do not transfer data between the host and the
1871          * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1872          * In order to be more cautious use controller's max_hw_sectors value
1873          * to configure the maximum sectors for the write-zeroes which is
1874          * configured based on the controller's MDTS field in the
1875          * nvme_init_identify() if available.
1876          */
1877         if (ns->ctrl->max_hw_sectors == UINT_MAX)
1878                 max_blocks = (u64)USHRT_MAX + 1;
1879         else
1880                 max_blocks = ns->ctrl->max_hw_sectors + 1;
1881
1882         blk_queue_max_write_zeroes_sectors(disk->queue,
1883                                            nvme_lba_to_sect(ns, max_blocks));
1884 }
1885
1886 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1887 {
1888         return !uuid_is_null(&ids->uuid) ||
1889                 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1890                 memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1891 }
1892
1893 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1894 {
1895         return uuid_equal(&a->uuid, &b->uuid) &&
1896                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1897                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1898                 a->csi == b->csi;
1899 }
1900
1901 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1902                                  u32 *phys_bs, u32 *io_opt)
1903 {
1904         struct streams_directive_params s;
1905         int ret;
1906
1907         if (!ctrl->nr_streams)
1908                 return 0;
1909
1910         ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
1911         if (ret)
1912                 return ret;
1913
1914         ns->sws = le32_to_cpu(s.sws);
1915         ns->sgs = le16_to_cpu(s.sgs);
1916
1917         if (ns->sws) {
1918                 *phys_bs = ns->sws * (1 << ns->lba_shift);
1919                 if (ns->sgs)
1920                         *io_opt = *phys_bs * ns->sgs;
1921         }
1922
1923         return 0;
1924 }
1925
1926 static int nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1927 {
1928         struct nvme_ctrl *ctrl = ns->ctrl;
1929
1930         /*
1931          * The PI implementation requires the metadata size to be equal to the
1932          * t10 pi tuple size.
1933          */
1934         ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1935         if (ns->ms == sizeof(struct t10_pi_tuple))
1936                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1937         else
1938                 ns->pi_type = 0;
1939
1940         ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1941         if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1942                 return 0;
1943         if (ctrl->ops->flags & NVME_F_FABRICS) {
1944                 /*
1945                  * The NVMe over Fabrics specification only supports metadata as
1946                  * part of the extended data LBA.  We rely on HCA/HBA support to
1947                  * remap the separate metadata buffer from the block layer.
1948                  */
1949                 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1950                         return -EINVAL;
1951                 if (ctrl->max_integrity_segments)
1952                         ns->features |=
1953                                 (NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1954         } else {
1955                 /*
1956                  * For PCIe controllers, we can't easily remap the separate
1957                  * metadata buffer from the block layer and thus require a
1958                  * separate metadata buffer for block layer metadata/PI support.
1959                  * We allow extended LBAs for the passthrough interface, though.
1960                  */
1961                 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1962                         ns->features |= NVME_NS_EXT_LBAS;
1963                 else
1964                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1965         }
1966
1967         return 0;
1968 }
1969
1970 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1971                 struct request_queue *q)
1972 {
1973         bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1974
1975         if (ctrl->max_hw_sectors) {
1976                 u32 max_segments =
1977                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1978
1979                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1980                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1981                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1982         }
1983         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1984         blk_queue_dma_alignment(q, 7);
1985         blk_queue_write_cache(q, vwc, vwc);
1986 }
1987
1988 static void nvme_update_disk_info(struct gendisk *disk,
1989                 struct nvme_ns *ns, struct nvme_id_ns *id)
1990 {
1991         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1992         unsigned short bs = 1 << ns->lba_shift;
1993         u32 atomic_bs, phys_bs, io_opt = 0;
1994
1995         /*
1996          * The block layer can't support LBA sizes larger than the page size
1997          * yet, so catch this early and don't allow block I/O.
1998          */
1999         if (ns->lba_shift > PAGE_SHIFT) {
2000                 capacity = 0;
2001                 bs = (1 << 9);
2002         }
2003
2004         blk_integrity_unregister(disk);
2005
2006         atomic_bs = phys_bs = bs;
2007         nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
2008         if (id->nabo == 0) {
2009                 /*
2010                  * Bit 1 indicates whether NAWUPF is defined for this namespace
2011                  * and whether it should be used instead of AWUPF. If NAWUPF ==
2012                  * 0 then AWUPF must be used instead.
2013                  */
2014                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
2015                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
2016                 else
2017                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
2018         }
2019
2020         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
2021                 /* NPWG = Namespace Preferred Write Granularity */
2022                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
2023                 /* NOWS = Namespace Optimal Write Size */
2024                 io_opt = bs * (1 + le16_to_cpu(id->nows));
2025         }
2026
2027         blk_queue_logical_block_size(disk->queue, bs);
2028         /*
2029          * Linux filesystems assume writing a single physical block is
2030          * an atomic operation. Hence limit the physical block size to the
2031          * value of the Atomic Write Unit Power Fail parameter.
2032          */
2033         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
2034         blk_queue_io_min(disk->queue, phys_bs);
2035         blk_queue_io_opt(disk->queue, io_opt);
2036
2037         /*
2038          * Register a metadata profile for PI, or the plain non-integrity NVMe
2039          * metadata masquerading as Type 0 if supported, otherwise reject block
2040          * I/O to namespaces with metadata except when the namespace supports
2041          * PI, as it can strip/insert in that case.
2042          */
2043         if (ns->ms) {
2044                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
2045                     (ns->features & NVME_NS_METADATA_SUPPORTED))
2046                         nvme_init_integrity(disk, ns->ms, ns->pi_type,
2047                                             ns->ctrl->max_integrity_segments);
2048                 else if (!nvme_ns_has_pi(ns))
2049                         capacity = 0;
2050         }
2051
2052         set_capacity_revalidate_and_notify(disk, capacity, false);
2053
2054         nvme_config_discard(disk, ns);
2055         nvme_config_write_zeroes(disk, ns);
2056
2057         if (id->nsattr & NVME_NS_ATTR_RO)
2058                 set_disk_ro(disk, true);
2059         else
2060                 set_disk_ro(disk, false);
2061 }
2062
2063 static inline bool nvme_first_scan(struct gendisk *disk)
2064 {
2065         /* nvme_alloc_ns() scans the disk prior to adding it */
2066         return !(disk->flags & GENHD_FL_UP);
2067 }
2068
2069 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
2070 {
2071         struct nvme_ctrl *ctrl = ns->ctrl;
2072         u32 iob;
2073
2074         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2075             is_power_of_2(ctrl->max_hw_sectors))
2076                 iob = ctrl->max_hw_sectors;
2077         else
2078                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
2079
2080         if (!iob)
2081                 return;
2082
2083         if (!is_power_of_2(iob)) {
2084                 if (nvme_first_scan(ns->disk))
2085                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2086                                 ns->disk->disk_name, iob);
2087                 return;
2088         }
2089
2090         if (blk_queue_is_zoned(ns->disk->queue)) {
2091                 if (nvme_first_scan(ns->disk))
2092                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
2093                                 ns->disk->disk_name);
2094                 return;
2095         }
2096
2097         blk_queue_chunk_sectors(ns->queue, iob);
2098 }
2099
2100 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
2101 {
2102         unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2103         int ret;
2104
2105         blk_mq_freeze_queue(ns->disk->queue);
2106         ns->lba_shift = id->lbaf[lbaf].ds;
2107         nvme_set_queue_limits(ns->ctrl, ns->queue);
2108
2109         if (ns->head->ids.csi == NVME_CSI_ZNS) {
2110                 ret = nvme_update_zone_info(ns, lbaf);
2111                 if (ret)
2112                         goto out_unfreeze;
2113         }
2114
2115         ret = nvme_configure_metadata(ns, id);
2116         if (ret)
2117                 goto out_unfreeze;
2118         nvme_set_chunk_sectors(ns, id);
2119         nvme_update_disk_info(ns->disk, ns, id);
2120         blk_mq_unfreeze_queue(ns->disk->queue);
2121
2122         if (blk_queue_is_zoned(ns->queue)) {
2123                 ret = nvme_revalidate_zones(ns);
2124                 if (ret)
2125                         return ret;
2126         }
2127
2128 #ifdef CONFIG_NVME_MULTIPATH
2129         if (ns->head->disk) {
2130                 blk_mq_freeze_queue(ns->head->disk->queue);
2131                 nvme_update_disk_info(ns->head->disk, ns, id);
2132                 blk_stack_limits(&ns->head->disk->queue->limits,
2133                                  &ns->queue->limits, 0);
2134                 blk_queue_update_readahead(ns->head->disk->queue);
2135                 nvme_update_bdev_size(ns->head->disk);
2136                 blk_mq_unfreeze_queue(ns->head->disk->queue);
2137         }
2138 #endif
2139         return 0;
2140
2141 out_unfreeze:
2142         blk_mq_unfreeze_queue(ns->disk->queue);
2143         return ret;
2144 }
2145
2146 static char nvme_pr_type(enum pr_type type)
2147 {
2148         switch (type) {
2149         case PR_WRITE_EXCLUSIVE:
2150                 return 1;
2151         case PR_EXCLUSIVE_ACCESS:
2152                 return 2;
2153         case PR_WRITE_EXCLUSIVE_REG_ONLY:
2154                 return 3;
2155         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2156                 return 4;
2157         case PR_WRITE_EXCLUSIVE_ALL_REGS:
2158                 return 5;
2159         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2160                 return 6;
2161         default:
2162                 return 0;
2163         }
2164 };
2165
2166 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2167                                 u64 key, u64 sa_key, u8 op)
2168 {
2169         struct nvme_ns_head *head = NULL;
2170         struct nvme_ns *ns;
2171         struct nvme_command c;
2172         int srcu_idx, ret;
2173         u8 data[16] = { 0, };
2174
2175         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
2176         if (unlikely(!ns))
2177                 return -EWOULDBLOCK;
2178
2179         put_unaligned_le64(key, &data[0]);
2180         put_unaligned_le64(sa_key, &data[8]);
2181
2182         memset(&c, 0, sizeof(c));
2183         c.common.opcode = op;
2184         c.common.nsid = cpu_to_le32(ns->head->ns_id);
2185         c.common.cdw10 = cpu_to_le32(cdw10);
2186
2187         ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
2188         nvme_put_ns_from_disk(head, srcu_idx);
2189         return ret;
2190 }
2191
2192 static int nvme_pr_register(struct block_device *bdev, u64 old,
2193                 u64 new, unsigned flags)
2194 {
2195         u32 cdw10;
2196
2197         if (flags & ~PR_FL_IGNORE_KEY)
2198                 return -EOPNOTSUPP;
2199
2200         cdw10 = old ? 2 : 0;
2201         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2202         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2203         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2204 }
2205
2206 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2207                 enum pr_type type, unsigned flags)
2208 {
2209         u32 cdw10;
2210
2211         if (flags & ~PR_FL_IGNORE_KEY)
2212                 return -EOPNOTSUPP;
2213
2214         cdw10 = nvme_pr_type(type) << 8;
2215         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2216         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2217 }
2218
2219 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2220                 enum pr_type type, bool abort)
2221 {
2222         u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2223         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2224 }
2225
2226 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2227 {
2228         u32 cdw10 = 1 | (key ? 1 << 3 : 0);
2229         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2230 }
2231
2232 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2233 {
2234         u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
2235         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2236 }
2237
2238 static const struct pr_ops nvme_pr_ops = {
2239         .pr_register    = nvme_pr_register,
2240         .pr_reserve     = nvme_pr_reserve,
2241         .pr_release     = nvme_pr_release,
2242         .pr_preempt     = nvme_pr_preempt,
2243         .pr_clear       = nvme_pr_clear,
2244 };
2245
2246 #ifdef CONFIG_BLK_SED_OPAL
2247 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2248                 bool send)
2249 {
2250         struct nvme_ctrl *ctrl = data;
2251         struct nvme_command cmd;
2252
2253         memset(&cmd, 0, sizeof(cmd));
2254         if (send)
2255                 cmd.common.opcode = nvme_admin_security_send;
2256         else
2257                 cmd.common.opcode = nvme_admin_security_recv;
2258         cmd.common.nsid = 0;
2259         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2260         cmd.common.cdw11 = cpu_to_le32(len);
2261
2262         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2263                                       ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2264 }
2265 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2266 #endif /* CONFIG_BLK_SED_OPAL */
2267
2268 static const struct block_device_operations nvme_fops = {
2269         .owner          = THIS_MODULE,
2270         .ioctl          = nvme_ioctl,
2271         .compat_ioctl   = nvme_compat_ioctl,
2272         .open           = nvme_open,
2273         .release        = nvme_release,
2274         .getgeo         = nvme_getgeo,
2275         .report_zones   = nvme_report_zones,
2276         .pr_ops         = &nvme_pr_ops,
2277 };
2278
2279 #ifdef CONFIG_NVME_MULTIPATH
2280 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2281 {
2282         struct nvme_ns_head *head = bdev->bd_disk->private_data;
2283
2284         if (!kref_get_unless_zero(&head->ref))
2285                 return -ENXIO;
2286         return 0;
2287 }
2288
2289 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2290 {
2291         nvme_put_ns_head(disk->private_data);
2292 }
2293
2294 const struct block_device_operations nvme_ns_head_ops = {
2295         .owner          = THIS_MODULE,
2296         .submit_bio     = nvme_ns_head_submit_bio,
2297         .open           = nvme_ns_head_open,
2298         .release        = nvme_ns_head_release,
2299         .ioctl          = nvme_ioctl,
2300         .compat_ioctl   = nvme_compat_ioctl,
2301         .getgeo         = nvme_getgeo,
2302         .report_zones   = nvme_report_zones,
2303         .pr_ops         = &nvme_pr_ops,
2304 };
2305 #endif /* CONFIG_NVME_MULTIPATH */
2306
2307 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2308 {
2309         unsigned long timeout =
2310                 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2311         u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2312         int ret;
2313
2314         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2315                 if (csts == ~0)
2316                         return -ENODEV;
2317                 if ((csts & NVME_CSTS_RDY) == bit)
2318                         break;
2319
2320                 usleep_range(1000, 2000);
2321                 if (fatal_signal_pending(current))
2322                         return -EINTR;
2323                 if (time_after(jiffies, timeout)) {
2324                         dev_err(ctrl->device,
2325                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2326                                 enabled ? "initialisation" : "reset", csts);
2327                         return -ENODEV;
2328                 }
2329         }
2330
2331         return ret;
2332 }
2333
2334 /*
2335  * If the device has been passed off to us in an enabled state, just clear
2336  * the enabled bit.  The spec says we should set the 'shutdown notification
2337  * bits', but doing so may cause the device to complete commands to the
2338  * admin queue ... and we don't know what memory that might be pointing at!
2339  */
2340 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2341 {
2342         int ret;
2343
2344         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2345         ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2346
2347         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2348         if (ret)
2349                 return ret;
2350
2351         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2352                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2353
2354         return nvme_wait_ready(ctrl, ctrl->cap, false);
2355 }
2356 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2357
2358 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2359 {
2360         unsigned dev_page_min;
2361         int ret;
2362
2363         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2364         if (ret) {
2365                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2366                 return ret;
2367         }
2368         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2369
2370         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2371                 dev_err(ctrl->device,
2372                         "Minimum device page size %u too large for host (%u)\n",
2373                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2374                 return -ENODEV;
2375         }
2376
2377         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2378                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2379         else
2380                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2381         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2382         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2383         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2384         ctrl->ctrl_config |= NVME_CC_ENABLE;
2385
2386         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2387         if (ret)
2388                 return ret;
2389         return nvme_wait_ready(ctrl, ctrl->cap, true);
2390 }
2391 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2392
2393 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2394 {
2395         unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2396         u32 csts;
2397         int ret;
2398
2399         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2400         ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2401
2402         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2403         if (ret)
2404                 return ret;
2405
2406         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2407                 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2408                         break;
2409
2410                 msleep(100);
2411                 if (fatal_signal_pending(current))
2412                         return -EINTR;
2413                 if (time_after(jiffies, timeout)) {
2414                         dev_err(ctrl->device,
2415                                 "Device shutdown incomplete; abort shutdown\n");
2416                         return -ENODEV;
2417                 }
2418         }
2419
2420         return ret;
2421 }
2422 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2423
2424 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2425 {
2426         __le64 ts;
2427         int ret;
2428
2429         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2430                 return 0;
2431
2432         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2433         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2434                         NULL);
2435         if (ret)
2436                 dev_warn_once(ctrl->device,
2437                         "could not set timestamp (%d)\n", ret);
2438         return ret;
2439 }
2440
2441 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2442 {
2443         struct nvme_feat_host_behavior *host;
2444         int ret;
2445
2446         /* Don't bother enabling the feature if retry delay is not reported */
2447         if (!ctrl->crdt[0])
2448                 return 0;
2449
2450         host = kzalloc(sizeof(*host), GFP_KERNEL);
2451         if (!host)
2452                 return 0;
2453
2454         host->acre = NVME_ENABLE_ACRE;
2455         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2456                                 host, sizeof(*host), NULL);
2457         kfree(host);
2458         return ret;
2459 }
2460
2461 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2462 {
2463         /*
2464          * APST (Autonomous Power State Transition) lets us program a
2465          * table of power state transitions that the controller will
2466          * perform automatically.  We configure it with a simple
2467          * heuristic: we are willing to spend at most 2% of the time
2468          * transitioning between power states.  Therefore, when running
2469          * in any given state, we will enter the next lower-power
2470          * non-operational state after waiting 50 * (enlat + exlat)
2471          * microseconds, as long as that state's exit latency is under
2472          * the requested maximum latency.
2473          *
2474          * We will not autonomously enter any non-operational state for
2475          * which the total latency exceeds ps_max_latency_us.  Users
2476          * can set ps_max_latency_us to zero to turn off APST.
2477          */
2478
2479         unsigned apste;
2480         struct nvme_feat_auto_pst *table;
2481         u64 max_lat_us = 0;
2482         int max_ps = -1;
2483         int ret;
2484
2485         /*
2486          * If APST isn't supported or if we haven't been initialized yet,
2487          * then don't do anything.
2488          */
2489         if (!ctrl->apsta)
2490                 return 0;
2491
2492         if (ctrl->npss > 31) {
2493                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2494                 return 0;
2495         }
2496
2497         table = kzalloc(sizeof(*table), GFP_KERNEL);
2498         if (!table)
2499                 return 0;
2500
2501         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2502                 /* Turn off APST. */
2503                 apste = 0;
2504                 dev_dbg(ctrl->device, "APST disabled\n");
2505         } else {
2506                 __le64 target = cpu_to_le64(0);
2507                 int state;
2508
2509                 /*
2510                  * Walk through all states from lowest- to highest-power.
2511                  * According to the spec, lower-numbered states use more
2512                  * power.  NPSS, despite the name, is the index of the
2513                  * lowest-power state, not the number of states.
2514                  */
2515                 for (state = (int)ctrl->npss; state >= 0; state--) {
2516                         u64 total_latency_us, exit_latency_us, transition_ms;
2517
2518                         if (target)
2519                                 table->entries[state] = target;
2520
2521                         /*
2522                          * Don't allow transitions to the deepest state
2523                          * if it's quirked off.
2524                          */
2525                         if (state == ctrl->npss &&
2526                             (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2527                                 continue;
2528
2529                         /*
2530                          * Is this state a useful non-operational state for
2531                          * higher-power states to autonomously transition to?
2532                          */
2533                         if (!(ctrl->psd[state].flags &
2534                               NVME_PS_FLAGS_NON_OP_STATE))
2535                                 continue;
2536
2537                         exit_latency_us =
2538                                 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2539                         if (exit_latency_us > ctrl->ps_max_latency_us)
2540                                 continue;
2541
2542                         total_latency_us =
2543                                 exit_latency_us +
2544                                 le32_to_cpu(ctrl->psd[state].entry_lat);
2545
2546                         /*
2547                          * This state is good.  Use it as the APST idle
2548                          * target for higher power states.
2549                          */
2550                         transition_ms = total_latency_us + 19;
2551                         do_div(transition_ms, 20);
2552                         if (transition_ms > (1 << 24) - 1)
2553                                 transition_ms = (1 << 24) - 1;
2554
2555                         target = cpu_to_le64((state << 3) |
2556                                              (transition_ms << 8));
2557
2558                         if (max_ps == -1)
2559                                 max_ps = state;
2560
2561                         if (total_latency_us > max_lat_us)
2562                                 max_lat_us = total_latency_us;
2563                 }
2564
2565                 apste = 1;
2566
2567                 if (max_ps == -1) {
2568                         dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2569                 } else {
2570                         dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2571                                 max_ps, max_lat_us, (int)sizeof(*table), table);
2572                 }
2573         }
2574
2575         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2576                                 table, sizeof(*table), NULL);
2577         if (ret)
2578                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2579
2580         kfree(table);
2581         return ret;
2582 }
2583
2584 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2585 {
2586         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2587         u64 latency;
2588
2589         switch (val) {
2590         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2591         case PM_QOS_LATENCY_ANY:
2592                 latency = U64_MAX;
2593                 break;
2594
2595         default:
2596                 latency = val;
2597         }
2598
2599         if (ctrl->ps_max_latency_us != latency) {
2600                 ctrl->ps_max_latency_us = latency;
2601                 nvme_configure_apst(ctrl);
2602         }
2603 }
2604
2605 struct nvme_core_quirk_entry {
2606         /*
2607          * NVMe model and firmware strings are padded with spaces.  For
2608          * simplicity, strings in the quirk table are padded with NULLs
2609          * instead.
2610          */
2611         u16 vid;
2612         const char *mn;
2613         const char *fr;
2614         unsigned long quirks;
2615 };
2616
2617 static const struct nvme_core_quirk_entry core_quirks[] = {
2618         {
2619                 /*
2620                  * This Toshiba device seems to die using any APST states.  See:
2621                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2622                  */
2623                 .vid = 0x1179,
2624                 .mn = "THNSF5256GPUK TOSHIBA",
2625                 .quirks = NVME_QUIRK_NO_APST,
2626         },
2627         {
2628                 /*
2629                  * This LiteON CL1-3D*-Q11 firmware version has a race
2630                  * condition associated with actions related to suspend to idle
2631                  * LiteON has resolved the problem in future firmware
2632                  */
2633                 .vid = 0x14a4,
2634                 .fr = "22301111",
2635                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2636         }
2637 };
2638
2639 /* match is null-terminated but idstr is space-padded. */
2640 static bool string_matches(const char *idstr, const char *match, size_t len)
2641 {
2642         size_t matchlen;
2643
2644         if (!match)
2645                 return true;
2646
2647         matchlen = strlen(match);
2648         WARN_ON_ONCE(matchlen > len);
2649
2650         if (memcmp(idstr, match, matchlen))
2651                 return false;
2652
2653         for (; matchlen < len; matchlen++)
2654                 if (idstr[matchlen] != ' ')
2655                         return false;
2656
2657         return true;
2658 }
2659
2660 static bool quirk_matches(const struct nvme_id_ctrl *id,
2661                           const struct nvme_core_quirk_entry *q)
2662 {
2663         return q->vid == le16_to_cpu(id->vid) &&
2664                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2665                 string_matches(id->fr, q->fr, sizeof(id->fr));
2666 }
2667
2668 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2669                 struct nvme_id_ctrl *id)
2670 {
2671         size_t nqnlen;
2672         int off;
2673
2674         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2675                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2676                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2677                         strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2678                         return;
2679                 }
2680
2681                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2682                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2683         }
2684
2685         /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2686         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2687                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2688                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2689         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2690         off += sizeof(id->sn);
2691         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2692         off += sizeof(id->mn);
2693         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2694 }
2695
2696 static void nvme_release_subsystem(struct device *dev)
2697 {
2698         struct nvme_subsystem *subsys =
2699                 container_of(dev, struct nvme_subsystem, dev);
2700
2701         if (subsys->instance >= 0)
2702                 ida_simple_remove(&nvme_instance_ida, subsys->instance);
2703         kfree(subsys);
2704 }
2705
2706 static void nvme_destroy_subsystem(struct kref *ref)
2707 {
2708         struct nvme_subsystem *subsys =
2709                         container_of(ref, struct nvme_subsystem, ref);
2710
2711         mutex_lock(&nvme_subsystems_lock);
2712         list_del(&subsys->entry);
2713         mutex_unlock(&nvme_subsystems_lock);
2714
2715         ida_destroy(&subsys->ns_ida);
2716         device_del(&subsys->dev);
2717         put_device(&subsys->dev);
2718 }
2719
2720 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2721 {
2722         kref_put(&subsys->ref, nvme_destroy_subsystem);
2723 }
2724
2725 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2726 {
2727         struct nvme_subsystem *subsys;
2728
2729         lockdep_assert_held(&nvme_subsystems_lock);
2730
2731         /*
2732          * Fail matches for discovery subsystems. This results
2733          * in each discovery controller bound to a unique subsystem.
2734          * This avoids issues with validating controller values
2735          * that can only be true when there is a single unique subsystem.
2736          * There may be multiple and completely independent entities
2737          * that provide discovery controllers.
2738          */
2739         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2740                 return NULL;
2741
2742         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2743                 if (strcmp(subsys->subnqn, subsysnqn))
2744                         continue;
2745                 if (!kref_get_unless_zero(&subsys->ref))
2746                         continue;
2747                 return subsys;
2748         }
2749
2750         return NULL;
2751 }
2752
2753 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2754         struct device_attribute subsys_attr_##_name = \
2755                 __ATTR(_name, _mode, _show, NULL)
2756
2757 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2758                                     struct device_attribute *attr,
2759                                     char *buf)
2760 {
2761         struct nvme_subsystem *subsys =
2762                 container_of(dev, struct nvme_subsystem, dev);
2763
2764         return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2765 }
2766 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2767
2768 #define nvme_subsys_show_str_function(field)                            \
2769 static ssize_t subsys_##field##_show(struct device *dev,                \
2770                             struct device_attribute *attr, char *buf)   \
2771 {                                                                       \
2772         struct nvme_subsystem *subsys =                                 \
2773                 container_of(dev, struct nvme_subsystem, dev);          \
2774         return sprintf(buf, "%.*s\n",                                   \
2775                        (int)sizeof(subsys->field), subsys->field);      \
2776 }                                                                       \
2777 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2778
2779 nvme_subsys_show_str_function(model);
2780 nvme_subsys_show_str_function(serial);
2781 nvme_subsys_show_str_function(firmware_rev);
2782
2783 static struct attribute *nvme_subsys_attrs[] = {
2784         &subsys_attr_model.attr,
2785         &subsys_attr_serial.attr,
2786         &subsys_attr_firmware_rev.attr,
2787         &subsys_attr_subsysnqn.attr,
2788 #ifdef CONFIG_NVME_MULTIPATH
2789         &subsys_attr_iopolicy.attr,
2790 #endif
2791         NULL,
2792 };
2793
2794 static struct attribute_group nvme_subsys_attrs_group = {
2795         .attrs = nvme_subsys_attrs,
2796 };
2797
2798 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2799         &nvme_subsys_attrs_group,
2800         NULL,
2801 };
2802
2803 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2804                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2805 {
2806         struct nvme_ctrl *tmp;
2807
2808         lockdep_assert_held(&nvme_subsystems_lock);
2809
2810         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2811                 if (nvme_state_terminal(tmp))
2812                         continue;
2813
2814                 if (tmp->cntlid == ctrl->cntlid) {
2815                         dev_err(ctrl->device,
2816                                 "Duplicate cntlid %u with %s, rejecting\n",
2817                                 ctrl->cntlid, dev_name(tmp->device));
2818                         return false;
2819                 }
2820
2821                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2822                     (ctrl->opts && ctrl->opts->discovery_nqn))
2823                         continue;
2824
2825                 dev_err(ctrl->device,
2826                         "Subsystem does not support multiple controllers\n");
2827                 return false;
2828         }
2829
2830         return true;
2831 }
2832
2833 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2834 {
2835         struct nvme_subsystem *subsys, *found;
2836         int ret;
2837
2838         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2839         if (!subsys)
2840                 return -ENOMEM;
2841
2842         subsys->instance = -1;
2843         mutex_init(&subsys->lock);
2844         kref_init(&subsys->ref);
2845         INIT_LIST_HEAD(&subsys->ctrls);
2846         INIT_LIST_HEAD(&subsys->nsheads);
2847         nvme_init_subnqn(subsys, ctrl, id);
2848         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2849         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2850         memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2851         subsys->vendor_id = le16_to_cpu(id->vid);
2852         subsys->cmic = id->cmic;
2853         subsys->awupf = le16_to_cpu(id->awupf);
2854 #ifdef CONFIG_NVME_MULTIPATH
2855         subsys->iopolicy = NVME_IOPOLICY_NUMA;
2856 #endif
2857
2858         subsys->dev.class = nvme_subsys_class;
2859         subsys->dev.release = nvme_release_subsystem;
2860         subsys->dev.groups = nvme_subsys_attrs_groups;
2861         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2862         device_initialize(&subsys->dev);
2863
2864         mutex_lock(&nvme_subsystems_lock);
2865         found = __nvme_find_get_subsystem(subsys->subnqn);
2866         if (found) {
2867                 put_device(&subsys->dev);
2868                 subsys = found;
2869
2870                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2871                         ret = -EINVAL;
2872                         goto out_put_subsystem;
2873                 }
2874         } else {
2875                 ret = device_add(&subsys->dev);
2876                 if (ret) {
2877                         dev_err(ctrl->device,
2878                                 "failed to register subsystem device.\n");
2879                         put_device(&subsys->dev);
2880                         goto out_unlock;
2881                 }
2882                 ida_init(&subsys->ns_ida);
2883                 list_add_tail(&subsys->entry, &nvme_subsystems);
2884         }
2885
2886         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2887                                 dev_name(ctrl->device));
2888         if (ret) {
2889                 dev_err(ctrl->device,
2890                         "failed to create sysfs link from subsystem.\n");
2891                 goto out_put_subsystem;
2892         }
2893
2894         if (!found)
2895                 subsys->instance = ctrl->instance;
2896         ctrl->subsys = subsys;
2897         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2898         mutex_unlock(&nvme_subsystems_lock);
2899         return 0;
2900
2901 out_put_subsystem:
2902         nvme_put_subsystem(subsys);
2903 out_unlock:
2904         mutex_unlock(&nvme_subsystems_lock);
2905         return ret;
2906 }
2907
2908 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2909                 void *log, size_t size, u64 offset)
2910 {
2911         struct nvme_command c = { };
2912         u32 dwlen = nvme_bytes_to_numd(size);
2913
2914         c.get_log_page.opcode = nvme_admin_get_log_page;
2915         c.get_log_page.nsid = cpu_to_le32(nsid);
2916         c.get_log_page.lid = log_page;
2917         c.get_log_page.lsp = lsp;
2918         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2919         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2920         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2921         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2922         c.get_log_page.csi = csi;
2923
2924         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2925 }
2926
2927 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2928                                 struct nvme_effects_log **log)
2929 {
2930         struct nvme_cel *cel = xa_load(&ctrl->cels, csi);
2931         int ret;
2932
2933         if (cel)
2934                 goto out;
2935
2936         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
2937         if (!cel)
2938                 return -ENOMEM;
2939
2940         ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
2941                         &cel->log, sizeof(cel->log), 0);
2942         if (ret) {
2943                 kfree(cel);
2944                 return ret;
2945         }
2946
2947         cel->csi = csi;
2948         xa_store(&ctrl->cels, cel->csi, cel, GFP_KERNEL);
2949 out:
2950         *log = &cel->log;
2951         return 0;
2952 }
2953
2954 /*
2955  * Initialize the cached copies of the Identify data and various controller
2956  * register in our nvme_ctrl structure.  This should be called as soon as
2957  * the admin queue is fully up and running.
2958  */
2959 int nvme_init_identify(struct nvme_ctrl *ctrl)
2960 {
2961         struct nvme_id_ctrl *id;
2962         int ret, page_shift;
2963         u32 max_hw_sectors;
2964         bool prev_apst_enabled;
2965
2966         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
2967         if (ret) {
2968                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2969                 return ret;
2970         }
2971         page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2972         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
2973
2974         if (ctrl->vs >= NVME_VS(1, 1, 0))
2975                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
2976
2977         ret = nvme_identify_ctrl(ctrl, &id);
2978         if (ret) {
2979                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2980                 return -EIO;
2981         }
2982
2983         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2984                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
2985                 if (ret < 0)
2986                         goto out_free;
2987         }
2988
2989         if (!(ctrl->ops->flags & NVME_F_FABRICS))
2990                 ctrl->cntlid = le16_to_cpu(id->cntlid);
2991
2992         if (!ctrl->identified) {
2993                 int i;
2994
2995                 ret = nvme_init_subsystem(ctrl, id);
2996                 if (ret)
2997                         goto out_free;
2998
2999                 /*
3000                  * Check for quirks.  Quirk can depend on firmware version,
3001                  * so, in principle, the set of quirks present can change
3002                  * across a reset.  As a possible future enhancement, we
3003                  * could re-scan for quirks every time we reinitialize
3004                  * the device, but we'd have to make sure that the driver
3005                  * behaves intelligently if the quirks change.
3006                  */
3007                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3008                         if (quirk_matches(id, &core_quirks[i]))
3009                                 ctrl->quirks |= core_quirks[i].quirks;
3010                 }
3011         }
3012
3013         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3014                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3015                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3016         }
3017
3018         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3019         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3020         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3021
3022         ctrl->oacs = le16_to_cpu(id->oacs);
3023         ctrl->oncs = le16_to_cpu(id->oncs);
3024         ctrl->mtfa = le16_to_cpu(id->mtfa);
3025         ctrl->oaes = le32_to_cpu(id->oaes);
3026         ctrl->wctemp = le16_to_cpu(id->wctemp);
3027         ctrl->cctemp = le16_to_cpu(id->cctemp);
3028
3029         atomic_set(&ctrl->abort_limit, id->acl + 1);
3030         ctrl->vwc = id->vwc;
3031         if (id->mdts)
3032                 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
3033         else
3034                 max_hw_sectors = UINT_MAX;
3035         ctrl->max_hw_sectors =
3036                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3037
3038         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3039         ctrl->sgls = le32_to_cpu(id->sgls);
3040         ctrl->kas = le16_to_cpu(id->kas);
3041         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3042         ctrl->ctratt = le32_to_cpu(id->ctratt);
3043
3044         if (id->rtd3e) {
3045                 /* us -> s */
3046                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3047
3048                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3049                                                  shutdown_timeout, 60);
3050
3051                 if (ctrl->shutdown_timeout != shutdown_timeout)
3052                         dev_info(ctrl->device,
3053                                  "Shutdown timeout set to %u seconds\n",
3054                                  ctrl->shutdown_timeout);
3055         } else
3056                 ctrl->shutdown_timeout = shutdown_timeout;
3057
3058         ctrl->npss = id->npss;
3059         ctrl->apsta = id->apsta;
3060         prev_apst_enabled = ctrl->apst_enabled;
3061         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3062                 if (force_apst && id->apsta) {
3063                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3064                         ctrl->apst_enabled = true;
3065                 } else {
3066                         ctrl->apst_enabled = false;
3067                 }
3068         } else {
3069                 ctrl->apst_enabled = id->apsta;
3070         }
3071         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3072
3073         if (ctrl->ops->flags & NVME_F_FABRICS) {
3074                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3075                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3076                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3077                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3078
3079                 /*
3080                  * In fabrics we need to verify the cntlid matches the
3081                  * admin connect
3082                  */
3083                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3084                         dev_err(ctrl->device,
3085                                 "Mismatching cntlid: Connect %u vs Identify "
3086                                 "%u, rejecting\n",
3087                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3088                         ret = -EINVAL;
3089                         goto out_free;
3090                 }
3091
3092                 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
3093                         dev_err(ctrl->device,
3094                                 "keep-alive support is mandatory for fabrics\n");
3095                         ret = -EINVAL;
3096                         goto out_free;
3097                 }
3098         } else {
3099                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3100                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3101                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3102                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3103         }
3104
3105         ret = nvme_mpath_init(ctrl, id);
3106         kfree(id);
3107
3108         if (ret < 0)
3109                 return ret;
3110
3111         if (ctrl->apst_enabled && !prev_apst_enabled)
3112                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3113         else if (!ctrl->apst_enabled && prev_apst_enabled)
3114                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3115
3116         ret = nvme_configure_apst(ctrl);
3117         if (ret < 0)
3118                 return ret;
3119
3120         ret = nvme_configure_timestamp(ctrl);
3121         if (ret < 0)
3122                 return ret;
3123
3124         ret = nvme_configure_directives(ctrl);
3125         if (ret < 0)
3126                 return ret;
3127
3128         ret = nvme_configure_acre(ctrl);
3129         if (ret < 0)
3130                 return ret;
3131
3132         if (!ctrl->identified) {
3133                 ret = nvme_hwmon_init(ctrl);
3134                 if (ret < 0)
3135                         return ret;
3136         }
3137
3138         ctrl->identified = true;
3139
3140         return 0;
3141
3142 out_free:
3143         kfree(id);
3144         return ret;
3145 }
3146 EXPORT_SYMBOL_GPL(nvme_init_identify);
3147
3148 static int nvme_dev_open(struct inode *inode, struct file *file)
3149 {
3150         struct nvme_ctrl *ctrl =
3151                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3152
3153         switch (ctrl->state) {
3154         case NVME_CTRL_LIVE:
3155                 break;
3156         default:
3157                 return -EWOULDBLOCK;
3158         }
3159
3160         nvme_get_ctrl(ctrl);
3161         if (!try_module_get(ctrl->ops->module)) {
3162                 nvme_put_ctrl(ctrl);
3163                 return -EINVAL;
3164         }
3165
3166         file->private_data = ctrl;
3167         return 0;
3168 }
3169
3170 static int nvme_dev_release(struct inode *inode, struct file *file)
3171 {
3172         struct nvme_ctrl *ctrl =
3173                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3174
3175         module_put(ctrl->ops->module);
3176         nvme_put_ctrl(ctrl);
3177         return 0;
3178 }
3179
3180 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
3181 {
3182         struct nvme_ns *ns;
3183         int ret;
3184
3185         down_read(&ctrl->namespaces_rwsem);
3186         if (list_empty(&ctrl->namespaces)) {
3187                 ret = -ENOTTY;
3188                 goto out_unlock;
3189         }
3190
3191         ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
3192         if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
3193                 dev_warn(ctrl->device,
3194                         "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
3195                 ret = -EINVAL;
3196                 goto out_unlock;
3197         }
3198
3199         dev_warn(ctrl->device,
3200                 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
3201         kref_get(&ns->kref);
3202         up_read(&ctrl->namespaces_rwsem);
3203
3204         ret = nvme_user_cmd(ctrl, ns, argp);
3205         nvme_put_ns(ns);
3206         return ret;
3207
3208 out_unlock:
3209         up_read(&ctrl->namespaces_rwsem);
3210         return ret;
3211 }
3212
3213 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
3214                 unsigned long arg)
3215 {
3216         struct nvme_ctrl *ctrl = file->private_data;
3217         void __user *argp = (void __user *)arg;
3218
3219         switch (cmd) {
3220         case NVME_IOCTL_ADMIN_CMD:
3221                 return nvme_user_cmd(ctrl, NULL, argp);
3222         case NVME_IOCTL_ADMIN64_CMD:
3223                 return nvme_user_cmd64(ctrl, NULL, argp);
3224         case NVME_IOCTL_IO_CMD:
3225                 return nvme_dev_user_cmd(ctrl, argp);
3226         case NVME_IOCTL_RESET:
3227                 dev_warn(ctrl->device, "resetting controller\n");
3228                 return nvme_reset_ctrl_sync(ctrl);
3229         case NVME_IOCTL_SUBSYS_RESET:
3230                 return nvme_reset_subsystem(ctrl);
3231         case NVME_IOCTL_RESCAN:
3232                 nvme_queue_scan(ctrl);
3233                 return 0;
3234         default:
3235                 return -ENOTTY;
3236         }
3237 }
3238
3239 static const struct file_operations nvme_dev_fops = {
3240         .owner          = THIS_MODULE,
3241         .open           = nvme_dev_open,
3242         .release        = nvme_dev_release,
3243         .unlocked_ioctl = nvme_dev_ioctl,
3244         .compat_ioctl   = compat_ptr_ioctl,
3245 };
3246
3247 static ssize_t nvme_sysfs_reset(struct device *dev,
3248                                 struct device_attribute *attr, const char *buf,
3249                                 size_t count)
3250 {
3251         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3252         int ret;
3253
3254         ret = nvme_reset_ctrl_sync(ctrl);
3255         if (ret < 0)
3256                 return ret;
3257         return count;
3258 }
3259 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3260
3261 static ssize_t nvme_sysfs_rescan(struct device *dev,
3262                                 struct device_attribute *attr, const char *buf,
3263                                 size_t count)
3264 {
3265         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3266
3267         nvme_queue_scan(ctrl);
3268         return count;
3269 }
3270 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3271
3272 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3273 {
3274         struct gendisk *disk = dev_to_disk(dev);
3275
3276         if (disk->fops == &nvme_fops)
3277                 return nvme_get_ns_from_dev(dev)->head;
3278         else
3279                 return disk->private_data;
3280 }
3281
3282 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3283                 char *buf)
3284 {
3285         struct nvme_ns_head *head = dev_to_ns_head(dev);
3286         struct nvme_ns_ids *ids = &head->ids;
3287         struct nvme_subsystem *subsys = head->subsys;
3288         int serial_len = sizeof(subsys->serial);
3289         int model_len = sizeof(subsys->model);
3290
3291         if (!uuid_is_null(&ids->uuid))
3292                 return sprintf(buf, "uuid.%pU\n", &ids->uuid);
3293
3294         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3295                 return sprintf(buf, "eui.%16phN\n", ids->nguid);
3296
3297         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3298                 return sprintf(buf, "eui.%8phN\n", ids->eui64);
3299
3300         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3301                                   subsys->serial[serial_len - 1] == '\0'))
3302                 serial_len--;
3303         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3304                                  subsys->model[model_len - 1] == '\0'))
3305                 model_len--;
3306
3307         return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3308                 serial_len, subsys->serial, model_len, subsys->model,
3309                 head->ns_id);
3310 }
3311 static DEVICE_ATTR_RO(wwid);
3312
3313 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3314                 char *buf)
3315 {
3316         return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3317 }
3318 static DEVICE_ATTR_RO(nguid);
3319
3320 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3321                 char *buf)
3322 {
3323         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3324
3325         /* For backward compatibility expose the NGUID to userspace if
3326          * we have no UUID set
3327          */
3328         if (uuid_is_null(&ids->uuid)) {
3329                 printk_ratelimited(KERN_WARNING
3330                                    "No UUID available providing old NGUID\n");
3331                 return sprintf(buf, "%pU\n", ids->nguid);
3332         }
3333         return sprintf(buf, "%pU\n", &ids->uuid);
3334 }
3335 static DEVICE_ATTR_RO(uuid);
3336
3337 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3338                 char *buf)
3339 {
3340         return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3341 }
3342 static DEVICE_ATTR_RO(eui);
3343
3344 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3345                 char *buf)
3346 {
3347         return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3348 }
3349 static DEVICE_ATTR_RO(nsid);
3350
3351 static struct attribute *nvme_ns_id_attrs[] = {
3352         &dev_attr_wwid.attr,
3353         &dev_attr_uuid.attr,
3354         &dev_attr_nguid.attr,
3355         &dev_attr_eui.attr,
3356         &dev_attr_nsid.attr,
3357 #ifdef CONFIG_NVME_MULTIPATH
3358         &dev_attr_ana_grpid.attr,
3359         &dev_attr_ana_state.attr,
3360 #endif
3361         NULL,
3362 };
3363
3364 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3365                 struct attribute *a, int n)
3366 {
3367         struct device *dev = container_of(kobj, struct device, kobj);
3368         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3369
3370         if (a == &dev_attr_uuid.attr) {
3371                 if (uuid_is_null(&ids->uuid) &&
3372                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3373                         return 0;
3374         }
3375         if (a == &dev_attr_nguid.attr) {
3376                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3377                         return 0;
3378         }
3379         if (a == &dev_attr_eui.attr) {
3380                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3381                         return 0;
3382         }
3383 #ifdef CONFIG_NVME_MULTIPATH
3384         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3385                 if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3386                         return 0;
3387                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3388                         return 0;
3389         }
3390 #endif
3391         return a->mode;
3392 }
3393
3394 static const struct attribute_group nvme_ns_id_attr_group = {
3395         .attrs          = nvme_ns_id_attrs,
3396         .is_visible     = nvme_ns_id_attrs_are_visible,
3397 };
3398
3399 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3400         &nvme_ns_id_attr_group,
3401 #ifdef CONFIG_NVM
3402         &nvme_nvm_attr_group,
3403 #endif
3404         NULL,
3405 };
3406
3407 #define nvme_show_str_function(field)                                           \
3408 static ssize_t  field##_show(struct device *dev,                                \
3409                             struct device_attribute *attr, char *buf)           \
3410 {                                                                               \
3411         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3412         return sprintf(buf, "%.*s\n",                                           \
3413                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3414 }                                                                               \
3415 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3416
3417 nvme_show_str_function(model);
3418 nvme_show_str_function(serial);
3419 nvme_show_str_function(firmware_rev);
3420
3421 #define nvme_show_int_function(field)                                           \
3422 static ssize_t  field##_show(struct device *dev,                                \
3423                             struct device_attribute *attr, char *buf)           \
3424 {                                                                               \
3425         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3426         return sprintf(buf, "%d\n", ctrl->field);       \
3427 }                                                                               \
3428 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3429
3430 nvme_show_int_function(cntlid);
3431 nvme_show_int_function(numa_node);
3432 nvme_show_int_function(queue_count);
3433 nvme_show_int_function(sqsize);
3434
3435 static ssize_t nvme_sysfs_delete(struct device *dev,
3436                                 struct device_attribute *attr, const char *buf,
3437                                 size_t count)
3438 {
3439         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3440
3441         if (device_remove_file_self(dev, attr))
3442                 nvme_delete_ctrl_sync(ctrl);
3443         return count;
3444 }
3445 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3446
3447 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3448                                          struct device_attribute *attr,
3449                                          char *buf)
3450 {
3451         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3452
3453         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3454 }
3455 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3456
3457 static ssize_t nvme_sysfs_show_state(struct device *dev,
3458                                      struct device_attribute *attr,
3459                                      char *buf)
3460 {
3461         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3462         static const char *const state_name[] = {
3463                 [NVME_CTRL_NEW]         = "new",
3464                 [NVME_CTRL_LIVE]        = "live",
3465                 [NVME_CTRL_RESETTING]   = "resetting",
3466                 [NVME_CTRL_CONNECTING]  = "connecting",
3467                 [NVME_CTRL_DELETING]    = "deleting",
3468                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3469                 [NVME_CTRL_DEAD]        = "dead",
3470         };
3471
3472         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3473             state_name[ctrl->state])
3474                 return sprintf(buf, "%s\n", state_name[ctrl->state]);
3475
3476         return sprintf(buf, "unknown state\n");
3477 }
3478
3479 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3480
3481 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3482                                          struct device_attribute *attr,
3483                                          char *buf)
3484 {
3485         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3486
3487         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3488 }
3489 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3490
3491 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3492                                         struct device_attribute *attr,
3493                                         char *buf)
3494 {
3495         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3496
3497         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
3498 }
3499 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3500
3501 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3502                                         struct device_attribute *attr,
3503                                         char *buf)
3504 {
3505         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3506
3507         return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
3508 }
3509 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3510
3511 static ssize_t nvme_sysfs_show_address(struct device *dev,
3512                                          struct device_attribute *attr,
3513                                          char *buf)
3514 {
3515         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3516
3517         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3518 }
3519 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3520
3521 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3522                 struct device_attribute *attr, char *buf)
3523 {
3524         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3525         struct nvmf_ctrl_options *opts = ctrl->opts;
3526
3527         if (ctrl->opts->max_reconnects == -1)
3528                 return sprintf(buf, "off\n");
3529         return sprintf(buf, "%d\n",
3530                         opts->max_reconnects * opts->reconnect_delay);
3531 }
3532
3533 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3534                 struct device_attribute *attr, const char *buf, size_t count)
3535 {
3536         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3537         struct nvmf_ctrl_options *opts = ctrl->opts;
3538         int ctrl_loss_tmo, err;
3539
3540         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3541         if (err)
3542                 return -EINVAL;
3543
3544         else if (ctrl_loss_tmo < 0)
3545                 opts->max_reconnects = -1;
3546         else
3547                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3548                                                 opts->reconnect_delay);
3549         return count;
3550 }
3551 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3552         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3553
3554 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3555                 struct device_attribute *attr, char *buf)
3556 {
3557         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3558
3559         if (ctrl->opts->reconnect_delay == -1)
3560                 return sprintf(buf, "off\n");
3561         return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay);
3562 }
3563
3564 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3565                 struct device_attribute *attr, const char *buf, size_t count)
3566 {
3567         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3568         unsigned int v;
3569         int err;
3570
3571         err = kstrtou32(buf, 10, &v);
3572         if (err)
3573                 return err;
3574
3575         ctrl->opts->reconnect_delay = v;
3576         return count;
3577 }
3578 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3579         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3580
3581 static struct attribute *nvme_dev_attrs[] = {
3582         &dev_attr_reset_controller.attr,
3583         &dev_attr_rescan_controller.attr,
3584         &dev_attr_model.attr,
3585         &dev_attr_serial.attr,
3586         &dev_attr_firmware_rev.attr,
3587         &dev_attr_cntlid.attr,
3588         &dev_attr_delete_controller.attr,
3589         &dev_attr_transport.attr,
3590         &dev_attr_subsysnqn.attr,
3591         &dev_attr_address.attr,
3592         &dev_attr_state.attr,
3593         &dev_attr_numa_node.attr,
3594         &dev_attr_queue_count.attr,
3595         &dev_attr_sqsize.attr,
3596         &dev_attr_hostnqn.attr,
3597         &dev_attr_hostid.attr,
3598         &dev_attr_ctrl_loss_tmo.attr,
3599         &dev_attr_reconnect_delay.attr,
3600         NULL
3601 };
3602
3603 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3604                 struct attribute *a, int n)
3605 {
3606         struct device *dev = container_of(kobj, struct device, kobj);
3607         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3608
3609         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3610                 return 0;
3611         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3612                 return 0;
3613         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3614                 return 0;
3615         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3616                 return 0;
3617         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3618                 return 0;
3619         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3620                 return 0;
3621
3622         return a->mode;
3623 }
3624
3625 static struct attribute_group nvme_dev_attrs_group = {
3626         .attrs          = nvme_dev_attrs,
3627         .is_visible     = nvme_dev_attrs_are_visible,
3628 };
3629
3630 static const struct attribute_group *nvme_dev_attr_groups[] = {
3631         &nvme_dev_attrs_group,
3632         NULL,
3633 };
3634
3635 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
3636                 unsigned nsid)
3637 {
3638         struct nvme_ns_head *h;
3639
3640         lockdep_assert_held(&subsys->lock);
3641
3642         list_for_each_entry(h, &subsys->nsheads, entry) {
3643                 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3644                         return h;
3645         }
3646
3647         return NULL;
3648 }
3649
3650 static int __nvme_check_ids(struct nvme_subsystem *subsys,
3651                 struct nvme_ns_head *new)
3652 {
3653         struct nvme_ns_head *h;
3654
3655         lockdep_assert_held(&subsys->lock);
3656
3657         list_for_each_entry(h, &subsys->nsheads, entry) {
3658                 if (nvme_ns_ids_valid(&new->ids) &&
3659                     nvme_ns_ids_equal(&new->ids, &h->ids))
3660                         return -EINVAL;
3661         }
3662
3663         return 0;
3664 }
3665
3666 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3667                 unsigned nsid, struct nvme_ns_ids *ids)
3668 {
3669         struct nvme_ns_head *head;
3670         size_t size = sizeof(*head);
3671         int ret = -ENOMEM;
3672
3673 #ifdef CONFIG_NVME_MULTIPATH
3674         size += num_possible_nodes() * sizeof(struct nvme_ns *);
3675 #endif
3676
3677         head = kzalloc(size, GFP_KERNEL);
3678         if (!head)
3679                 goto out;
3680         ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3681         if (ret < 0)
3682                 goto out_free_head;
3683         head->instance = ret;
3684         INIT_LIST_HEAD(&head->list);
3685         ret = init_srcu_struct(&head->srcu);
3686         if (ret)
3687                 goto out_ida_remove;
3688         head->subsys = ctrl->subsys;
3689         head->ns_id = nsid;
3690         head->ids = *ids;
3691         kref_init(&head->ref);
3692
3693         ret = __nvme_check_ids(ctrl->subsys, head);
3694         if (ret) {
3695                 dev_err(ctrl->device,
3696                         "duplicate IDs for nsid %d\n", nsid);
3697                 goto out_cleanup_srcu;
3698         }
3699
3700         if (head->ids.csi) {
3701                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3702                 if (ret)
3703                         goto out_cleanup_srcu;
3704         } else
3705                 head->effects = ctrl->effects;
3706
3707         ret = nvme_mpath_alloc_disk(ctrl, head);
3708         if (ret)
3709                 goto out_cleanup_srcu;
3710
3711         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3712
3713         kref_get(&ctrl->subsys->ref);
3714
3715         return head;
3716 out_cleanup_srcu:
3717         cleanup_srcu_struct(&head->srcu);
3718 out_ida_remove:
3719         ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3720 out_free_head:
3721         kfree(head);
3722 out:
3723         if (ret > 0)
3724                 ret = blk_status_to_errno(nvme_error_status(ret));
3725         return ERR_PTR(ret);
3726 }
3727
3728 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3729                 struct nvme_ns_ids *ids, bool is_shared)
3730 {
3731         struct nvme_ctrl *ctrl = ns->ctrl;
3732         struct nvme_ns_head *head = NULL;
3733         int ret = 0;
3734
3735         mutex_lock(&ctrl->subsys->lock);
3736         head = nvme_find_ns_head(ctrl->subsys, nsid);
3737         if (!head) {
3738                 head = nvme_alloc_ns_head(ctrl, nsid, ids);
3739                 if (IS_ERR(head)) {
3740                         ret = PTR_ERR(head);
3741                         goto out_unlock;
3742                 }
3743                 head->shared = is_shared;
3744         } else {
3745                 ret = -EINVAL;
3746                 if (!is_shared || !head->shared) {
3747                         dev_err(ctrl->device,
3748                                 "Duplicate unshared namespace %d\n", nsid);
3749                         goto out_put_ns_head;
3750                 }
3751                 if (!nvme_ns_ids_equal(&head->ids, ids)) {
3752                         dev_err(ctrl->device,
3753                                 "IDs don't match for shared namespace %d\n",
3754                                         nsid);
3755                         goto out_put_ns_head;
3756                 }
3757         }
3758
3759         list_add_tail(&ns->siblings, &head->list);
3760         ns->head = head;
3761         mutex_unlock(&ctrl->subsys->lock);
3762         return 0;
3763
3764 out_put_ns_head:
3765         nvme_put_ns_head(head);
3766 out_unlock:
3767         mutex_unlock(&ctrl->subsys->lock);
3768         return ret;
3769 }
3770
3771 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3772 {
3773         struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3774         struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3775
3776         return nsa->head->ns_id - nsb->head->ns_id;
3777 }
3778
3779 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3780 {
3781         struct nvme_ns *ns, *ret = NULL;
3782
3783         down_read(&ctrl->namespaces_rwsem);
3784         list_for_each_entry(ns, &ctrl->namespaces, list) {
3785                 if (ns->head->ns_id == nsid) {
3786                         if (!kref_get_unless_zero(&ns->kref))
3787                                 continue;
3788                         ret = ns;
3789                         break;
3790                 }
3791                 if (ns->head->ns_id > nsid)
3792                         break;
3793         }
3794         up_read(&ctrl->namespaces_rwsem);
3795         return ret;
3796 }
3797 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3798
3799 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
3800                 struct nvme_ns_ids *ids)
3801 {
3802         struct nvme_ns *ns;
3803         struct gendisk *disk;
3804         struct nvme_id_ns *id;
3805         char disk_name[DISK_NAME_LEN];
3806         int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3807
3808         if (nvme_identify_ns(ctrl, nsid, ids, &id))
3809                 return;
3810
3811         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3812         if (!ns)
3813                 goto out_free_id;
3814
3815         ns->queue = blk_mq_init_queue(ctrl->tagset);
3816         if (IS_ERR(ns->queue))
3817                 goto out_free_ns;
3818
3819         if (ctrl->opts && ctrl->opts->data_digest)
3820                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3821
3822         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3823         if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3824                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3825
3826         ns->queue->queuedata = ns;
3827         ns->ctrl = ctrl;
3828         kref_init(&ns->kref);
3829
3830         ret = nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED);
3831         if (ret)
3832                 goto out_free_queue;
3833         nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3834
3835         disk = alloc_disk_node(0, node);
3836         if (!disk)
3837                 goto out_unlink_ns;
3838
3839         disk->fops = &nvme_fops;
3840         disk->private_data = ns;
3841         disk->queue = ns->queue;
3842         disk->flags = flags;
3843         memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3844         ns->disk = disk;
3845
3846         if (nvme_update_ns_info(ns, id))
3847                 goto out_put_disk;
3848
3849         if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3850                 ret = nvme_nvm_register(ns, disk_name, node);
3851                 if (ret) {
3852                         dev_warn(ctrl->device, "LightNVM init failure\n");
3853                         goto out_put_disk;
3854                 }
3855         }
3856
3857         down_write(&ctrl->namespaces_rwsem);
3858         list_add_tail(&ns->list, &ctrl->namespaces);
3859         up_write(&ctrl->namespaces_rwsem);
3860
3861         nvme_get_ctrl(ctrl);
3862
3863         device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3864
3865         nvme_mpath_add_disk(ns, id);
3866         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3867         kfree(id);
3868
3869         return;
3870  out_put_disk:
3871         /* prevent double queue cleanup */
3872         ns->disk->queue = NULL;
3873         put_disk(ns->disk);
3874  out_unlink_ns:
3875         mutex_lock(&ctrl->subsys->lock);
3876         list_del_rcu(&ns->siblings);
3877         if (list_empty(&ns->head->list))
3878                 list_del_init(&ns->head->entry);
3879         mutex_unlock(&ctrl->subsys->lock);
3880         nvme_put_ns_head(ns->head);
3881  out_free_queue:
3882         blk_cleanup_queue(ns->queue);
3883  out_free_ns:
3884         kfree(ns);
3885  out_free_id:
3886         kfree(id);
3887 }
3888
3889 static void nvme_ns_remove(struct nvme_ns *ns)
3890 {
3891         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3892                 return;
3893
3894         set_capacity(ns->disk, 0);
3895         nvme_fault_inject_fini(&ns->fault_inject);
3896
3897         mutex_lock(&ns->ctrl->subsys->lock);
3898         list_del_rcu(&ns->siblings);
3899         if (list_empty(&ns->head->list))
3900                 list_del_init(&ns->head->entry);
3901         mutex_unlock(&ns->ctrl->subsys->lock);
3902
3903         synchronize_rcu(); /* guarantee not available in head->list */
3904         nvme_mpath_clear_current_path(ns);
3905         synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3906
3907         if (ns->disk->flags & GENHD_FL_UP) {
3908                 del_gendisk(ns->disk);
3909                 blk_cleanup_queue(ns->queue);
3910                 if (blk_get_integrity(ns->disk))
3911                         blk_integrity_unregister(ns->disk);
3912         }
3913
3914         down_write(&ns->ctrl->namespaces_rwsem);
3915         list_del_init(&ns->list);
3916         up_write(&ns->ctrl->namespaces_rwsem);
3917
3918         nvme_mpath_check_last_path(ns);
3919         nvme_put_ns(ns);
3920 }
3921
3922 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
3923 {
3924         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
3925
3926         if (ns) {
3927                 nvme_ns_remove(ns);
3928                 nvme_put_ns(ns);
3929         }
3930 }
3931
3932 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_ids *ids)
3933 {
3934         struct nvme_id_ns *id;
3935         int ret = -ENODEV;
3936
3937         if (test_bit(NVME_NS_DEAD, &ns->flags))
3938                 goto out;
3939
3940         ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id);
3941         if (ret)
3942                 goto out;
3943
3944         ret = -ENODEV;
3945         if (!nvme_ns_ids_equal(&ns->head->ids, ids)) {
3946                 dev_err(ns->ctrl->device,
3947                         "identifiers changed for nsid %d\n", ns->head->ns_id);
3948                 goto out_free_id;
3949         }
3950
3951         ret = nvme_update_ns_info(ns, id);
3952
3953 out_free_id:
3954         kfree(id);
3955 out:
3956         /*
3957          * Only remove the namespace if we got a fatal error back from the
3958          * device, otherwise ignore the error and just move on.
3959          *
3960          * TODO: we should probably schedule a delayed retry here.
3961          */
3962         if (ret && ret != -ENOMEM && !(ret > 0 && !(ret & NVME_SC_DNR)))
3963                 nvme_ns_remove(ns);
3964         else
3965                 revalidate_disk_size(ns->disk, true);
3966 }
3967
3968 static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3969 {
3970         struct nvme_ns_ids ids = { };
3971         struct nvme_ns *ns;
3972
3973         if (nvme_identify_ns_descs(ctrl, nsid, &ids))
3974                 return;
3975
3976         ns = nvme_find_get_ns(ctrl, nsid);
3977         if (ns) {
3978                 nvme_validate_ns(ns, &ids);
3979                 nvme_put_ns(ns);
3980                 return;
3981         }
3982
3983         switch (ids.csi) {
3984         case NVME_CSI_NVM:
3985                 nvme_alloc_ns(ctrl, nsid, &ids);
3986                 break;
3987         case NVME_CSI_ZNS:
3988                 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
3989                         dev_warn(ctrl->device,
3990                                 "nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
3991                                 nsid);
3992                         break;
3993                 }
3994                 nvme_alloc_ns(ctrl, nsid, &ids);
3995                 break;
3996         default:
3997                 dev_warn(ctrl->device, "unknown csi %u for nsid %u\n",
3998                         ids.csi, nsid);
3999                 break;
4000         }
4001 }
4002
4003 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4004                                         unsigned nsid)
4005 {
4006         struct nvme_ns *ns, *next;
4007         LIST_HEAD(rm_list);
4008
4009         down_write(&ctrl->namespaces_rwsem);
4010         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4011                 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4012                         list_move_tail(&ns->list, &rm_list);
4013         }
4014         up_write(&ctrl->namespaces_rwsem);
4015
4016         list_for_each_entry_safe(ns, next, &rm_list, list)
4017                 nvme_ns_remove(ns);
4018
4019 }
4020
4021 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4022 {
4023         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4024         __le32 *ns_list;
4025         u32 prev = 0;
4026         int ret = 0, i;
4027
4028         if (nvme_ctrl_limited_cns(ctrl))
4029                 return -EOPNOTSUPP;
4030
4031         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4032         if (!ns_list)
4033                 return -ENOMEM;
4034
4035         for (;;) {
4036                 struct nvme_command cmd = {
4037                         .identify.opcode        = nvme_admin_identify,
4038                         .identify.cns           = NVME_ID_CNS_NS_ACTIVE_LIST,
4039                         .identify.nsid          = cpu_to_le32(prev),
4040                 };
4041
4042                 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4043                                             NVME_IDENTIFY_DATA_SIZE);
4044                 if (ret)
4045                         goto free;
4046
4047                 for (i = 0; i < nr_entries; i++) {
4048                         u32 nsid = le32_to_cpu(ns_list[i]);
4049
4050                         if (!nsid)      /* end of the list? */
4051                                 goto out;
4052                         nvme_validate_or_alloc_ns(ctrl, nsid);
4053                         while (++prev < nsid)
4054                                 nvme_ns_remove_by_nsid(ctrl, prev);
4055                 }
4056         }
4057  out:
4058         nvme_remove_invalid_namespaces(ctrl, prev);
4059  free:
4060         kfree(ns_list);
4061         return ret;
4062 }
4063
4064 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4065 {
4066         struct nvme_id_ctrl *id;
4067         u32 nn, i;
4068
4069         if (nvme_identify_ctrl(ctrl, &id))
4070                 return;
4071         nn = le32_to_cpu(id->nn);
4072         kfree(id);
4073
4074         for (i = 1; i <= nn; i++)
4075                 nvme_validate_or_alloc_ns(ctrl, i);
4076
4077         nvme_remove_invalid_namespaces(ctrl, nn);
4078 }
4079
4080 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4081 {
4082         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4083         __le32 *log;
4084         int error;
4085
4086         log = kzalloc(log_size, GFP_KERNEL);
4087         if (!log)
4088                 return;
4089
4090         /*
4091          * We need to read the log to clear the AEN, but we don't want to rely
4092          * on it for the changed namespace information as userspace could have
4093          * raced with us in reading the log page, which could cause us to miss
4094          * updates.
4095          */
4096         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4097                         NVME_CSI_NVM, log, log_size, 0);
4098         if (error)
4099                 dev_warn(ctrl->device,
4100                         "reading changed ns log failed: %d\n", error);
4101
4102         kfree(log);
4103 }
4104
4105 static void nvme_scan_work(struct work_struct *work)
4106 {
4107         struct nvme_ctrl *ctrl =
4108                 container_of(work, struct nvme_ctrl, scan_work);
4109
4110         /* No tagset on a live ctrl means IO queues could not created */
4111         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4112                 return;
4113
4114         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4115                 dev_info(ctrl->device, "rescanning namespaces.\n");
4116                 nvme_clear_changed_ns_log(ctrl);
4117         }
4118
4119         mutex_lock(&ctrl->scan_lock);
4120         if (nvme_scan_ns_list(ctrl) != 0)
4121                 nvme_scan_ns_sequential(ctrl);
4122         mutex_unlock(&ctrl->scan_lock);
4123
4124         down_write(&ctrl->namespaces_rwsem);
4125         list_sort(NULL, &ctrl->namespaces, ns_cmp);
4126         up_write(&ctrl->namespaces_rwsem);
4127 }
4128
4129 /*
4130  * This function iterates the namespace list unlocked to allow recovery from
4131  * controller failure. It is up to the caller to ensure the namespace list is
4132  * not modified by scan work while this function is executing.
4133  */
4134 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4135 {
4136         struct nvme_ns *ns, *next;
4137         LIST_HEAD(ns_list);
4138
4139         /*
4140          * make sure to requeue I/O to all namespaces as these
4141          * might result from the scan itself and must complete
4142          * for the scan_work to make progress
4143          */
4144         nvme_mpath_clear_ctrl_paths(ctrl);
4145
4146         /* prevent racing with ns scanning */
4147         flush_work(&ctrl->scan_work);
4148
4149         /*
4150          * The dead states indicates the controller was not gracefully
4151          * disconnected. In that case, we won't be able to flush any data while
4152          * removing the namespaces' disks; fail all the queues now to avoid
4153          * potentially having to clean up the failed sync later.
4154          */
4155         if (ctrl->state == NVME_CTRL_DEAD)
4156                 nvme_kill_queues(ctrl);
4157
4158         /* this is a no-op when called from the controller reset handler */
4159         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4160
4161         down_write(&ctrl->namespaces_rwsem);
4162         list_splice_init(&ctrl->namespaces, &ns_list);
4163         up_write(&ctrl->namespaces_rwsem);
4164
4165         list_for_each_entry_safe(ns, next, &ns_list, list)
4166                 nvme_ns_remove(ns);
4167 }
4168 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4169
4170 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4171 {
4172         struct nvme_ctrl *ctrl =
4173                 container_of(dev, struct nvme_ctrl, ctrl_device);
4174         struct nvmf_ctrl_options *opts = ctrl->opts;
4175         int ret;
4176
4177         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4178         if (ret)
4179                 return ret;
4180
4181         if (opts) {
4182                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4183                 if (ret)
4184                         return ret;
4185
4186                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4187                                 opts->trsvcid ?: "none");
4188                 if (ret)
4189                         return ret;
4190
4191                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4192                                 opts->host_traddr ?: "none");
4193         }
4194         return ret;
4195 }
4196
4197 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4198 {
4199         char *envp[2] = { NULL, NULL };
4200         u32 aen_result = ctrl->aen_result;
4201
4202         ctrl->aen_result = 0;
4203         if (!aen_result)
4204                 return;
4205
4206         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4207         if (!envp[0])
4208                 return;
4209         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4210         kfree(envp[0]);
4211 }
4212
4213 static void nvme_async_event_work(struct work_struct *work)
4214 {
4215         struct nvme_ctrl *ctrl =
4216                 container_of(work, struct nvme_ctrl, async_event_work);
4217
4218         nvme_aen_uevent(ctrl);
4219         ctrl->ops->submit_async_event(ctrl);
4220 }
4221
4222 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4223 {
4224
4225         u32 csts;
4226
4227         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4228                 return false;
4229
4230         if (csts == ~0)
4231                 return false;
4232
4233         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4234 }
4235
4236 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4237 {
4238         struct nvme_fw_slot_info_log *log;
4239
4240         log = kmalloc(sizeof(*log), GFP_KERNEL);
4241         if (!log)
4242                 return;
4243
4244         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4245                         log, sizeof(*log), 0))
4246                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4247         kfree(log);
4248 }
4249
4250 static void nvme_fw_act_work(struct work_struct *work)
4251 {
4252         struct nvme_ctrl *ctrl = container_of(work,
4253                                 struct nvme_ctrl, fw_act_work);
4254         unsigned long fw_act_timeout;
4255
4256         if (ctrl->mtfa)
4257                 fw_act_timeout = jiffies +
4258                                 msecs_to_jiffies(ctrl->mtfa * 100);
4259         else
4260                 fw_act_timeout = jiffies +
4261                                 msecs_to_jiffies(admin_timeout * 1000);
4262
4263         nvme_stop_queues(ctrl);
4264         while (nvme_ctrl_pp_status(ctrl)) {
4265                 if (time_after(jiffies, fw_act_timeout)) {
4266                         dev_warn(ctrl->device,
4267                                 "Fw activation timeout, reset controller\n");
4268                         nvme_try_sched_reset(ctrl);
4269                         return;
4270                 }
4271                 msleep(100);
4272         }
4273
4274         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4275                 return;
4276
4277         nvme_start_queues(ctrl);
4278         /* read FW slot information to clear the AER */
4279         nvme_get_fw_slot_info(ctrl);
4280 }
4281
4282 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4283 {
4284         u32 aer_notice_type = (result & 0xff00) >> 8;
4285
4286         trace_nvme_async_event(ctrl, aer_notice_type);
4287
4288         switch (aer_notice_type) {
4289         case NVME_AER_NOTICE_NS_CHANGED:
4290                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4291                 nvme_queue_scan(ctrl);
4292                 break;
4293         case NVME_AER_NOTICE_FW_ACT_STARTING:
4294                 /*
4295                  * We are (ab)using the RESETTING state to prevent subsequent
4296                  * recovery actions from interfering with the controller's
4297                  * firmware activation.
4298                  */
4299                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4300                         queue_work(nvme_wq, &ctrl->fw_act_work);
4301                 break;
4302 #ifdef CONFIG_NVME_MULTIPATH
4303         case NVME_AER_NOTICE_ANA:
4304                 if (!ctrl->ana_log_buf)
4305                         break;
4306                 queue_work(nvme_wq, &ctrl->ana_work);
4307                 break;
4308 #endif
4309         case NVME_AER_NOTICE_DISC_CHANGED:
4310                 ctrl->aen_result = result;
4311                 break;
4312         default:
4313                 dev_warn(ctrl->device, "async event result %08x\n", result);
4314         }
4315 }
4316
4317 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4318                 volatile union nvme_result *res)
4319 {
4320         u32 result = le32_to_cpu(res->u32);
4321         u32 aer_type = result & 0x07;
4322
4323         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4324                 return;
4325
4326         switch (aer_type) {
4327         case NVME_AER_NOTICE:
4328                 nvme_handle_aen_notice(ctrl, result);
4329                 break;
4330         case NVME_AER_ERROR:
4331         case NVME_AER_SMART:
4332         case NVME_AER_CSS:
4333         case NVME_AER_VS:
4334                 trace_nvme_async_event(ctrl, aer_type);
4335                 ctrl->aen_result = result;
4336                 break;
4337         default:
4338                 break;
4339         }
4340         queue_work(nvme_wq, &ctrl->async_event_work);
4341 }
4342 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4343
4344 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4345 {
4346         nvme_mpath_stop(ctrl);
4347         nvme_stop_keep_alive(ctrl);
4348         flush_work(&ctrl->async_event_work);
4349         cancel_work_sync(&ctrl->fw_act_work);
4350 }
4351 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4352
4353 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4354 {
4355         nvme_start_keep_alive(ctrl);
4356
4357         nvme_enable_aen(ctrl);
4358
4359         if (ctrl->queue_count > 1) {
4360                 nvme_queue_scan(ctrl);
4361                 nvme_start_queues(ctrl);
4362         }
4363 }
4364 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4365
4366 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4367 {
4368         nvme_fault_inject_fini(&ctrl->fault_inject);
4369         dev_pm_qos_hide_latency_tolerance(ctrl->device);
4370         cdev_device_del(&ctrl->cdev, ctrl->device);
4371         nvme_put_ctrl(ctrl);
4372 }
4373 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4374
4375 static void nvme_free_ctrl(struct device *dev)
4376 {
4377         struct nvme_ctrl *ctrl =
4378                 container_of(dev, struct nvme_ctrl, ctrl_device);
4379         struct nvme_subsystem *subsys = ctrl->subsys;
4380
4381         if (!subsys || ctrl->instance != subsys->instance)
4382                 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4383
4384         xa_destroy(&ctrl->cels);
4385
4386         nvme_mpath_uninit(ctrl);
4387         __free_page(ctrl->discard_page);
4388
4389         if (subsys) {
4390                 mutex_lock(&nvme_subsystems_lock);
4391                 list_del(&ctrl->subsys_entry);
4392                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4393                 mutex_unlock(&nvme_subsystems_lock);
4394         }
4395
4396         ctrl->ops->free_ctrl(ctrl);
4397
4398         if (subsys)
4399                 nvme_put_subsystem(subsys);
4400 }
4401
4402 /*
4403  * Initialize a NVMe controller structures.  This needs to be called during
4404  * earliest initialization so that we have the initialized structured around
4405  * during probing.
4406  */
4407 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4408                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4409 {
4410         int ret;
4411
4412         ctrl->state = NVME_CTRL_NEW;
4413         spin_lock_init(&ctrl->lock);
4414         mutex_init(&ctrl->scan_lock);
4415         INIT_LIST_HEAD(&ctrl->namespaces);
4416         xa_init(&ctrl->cels);
4417         init_rwsem(&ctrl->namespaces_rwsem);
4418         ctrl->dev = dev;
4419         ctrl->ops = ops;
4420         ctrl->quirks = quirks;
4421         ctrl->numa_node = NUMA_NO_NODE;
4422         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4423         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4424         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4425         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4426         init_waitqueue_head(&ctrl->state_wq);
4427
4428         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4429         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4430         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4431
4432         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4433                         PAGE_SIZE);
4434         ctrl->discard_page = alloc_page(GFP_KERNEL);
4435         if (!ctrl->discard_page) {
4436                 ret = -ENOMEM;
4437                 goto out;
4438         }
4439
4440         ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4441         if (ret < 0)
4442                 goto out;
4443         ctrl->instance = ret;
4444
4445         device_initialize(&ctrl->ctrl_device);
4446         ctrl->device = &ctrl->ctrl_device;
4447         ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4448         ctrl->device->class = nvme_class;
4449         ctrl->device->parent = ctrl->dev;
4450         ctrl->device->groups = nvme_dev_attr_groups;
4451         ctrl->device->release = nvme_free_ctrl;
4452         dev_set_drvdata(ctrl->device, ctrl);
4453         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4454         if (ret)
4455                 goto out_release_instance;
4456
4457         nvme_get_ctrl(ctrl);
4458         cdev_init(&ctrl->cdev, &nvme_dev_fops);
4459         ctrl->cdev.owner = ops->module;
4460         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4461         if (ret)
4462                 goto out_free_name;
4463
4464         /*
4465          * Initialize latency tolerance controls.  The sysfs files won't
4466          * be visible to userspace unless the device actually supports APST.
4467          */
4468         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4469         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4470                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4471
4472         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4473
4474         return 0;
4475 out_free_name:
4476         nvme_put_ctrl(ctrl);
4477         kfree_const(ctrl->device->kobj.name);
4478 out_release_instance:
4479         ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4480 out:
4481         if (ctrl->discard_page)
4482                 __free_page(ctrl->discard_page);
4483         return ret;
4484 }
4485 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4486
4487 /**
4488  * nvme_kill_queues(): Ends all namespace queues
4489  * @ctrl: the dead controller that needs to end
4490  *
4491  * Call this function when the driver determines it is unable to get the
4492  * controller in a state capable of servicing IO.
4493  */
4494 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4495 {
4496         struct nvme_ns *ns;
4497
4498         down_read(&ctrl->namespaces_rwsem);
4499
4500         /* Forcibly unquiesce queues to avoid blocking dispatch */
4501         if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4502                 blk_mq_unquiesce_queue(ctrl->admin_q);
4503
4504         list_for_each_entry(ns, &ctrl->namespaces, list)
4505                 nvme_set_queue_dying(ns);
4506
4507         up_read(&ctrl->namespaces_rwsem);
4508 }
4509 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4510
4511 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4512 {
4513         struct nvme_ns *ns;
4514
4515         down_read(&ctrl->namespaces_rwsem);
4516         list_for_each_entry(ns, &ctrl->namespaces, list)
4517                 blk_mq_unfreeze_queue(ns->queue);
4518         up_read(&ctrl->namespaces_rwsem);
4519 }
4520 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4521
4522 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4523 {
4524         struct nvme_ns *ns;
4525
4526         down_read(&ctrl->namespaces_rwsem);
4527         list_for_each_entry(ns, &ctrl->namespaces, list) {
4528                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4529                 if (timeout <= 0)
4530                         break;
4531         }
4532         up_read(&ctrl->namespaces_rwsem);
4533         return timeout;
4534 }
4535 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4536
4537 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4538 {
4539         struct nvme_ns *ns;
4540
4541         down_read(&ctrl->namespaces_rwsem);
4542         list_for_each_entry(ns, &ctrl->namespaces, list)
4543                 blk_mq_freeze_queue_wait(ns->queue);
4544         up_read(&ctrl->namespaces_rwsem);
4545 }
4546 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4547
4548 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4549 {
4550         struct nvme_ns *ns;
4551
4552         down_read(&ctrl->namespaces_rwsem);
4553         list_for_each_entry(ns, &ctrl->namespaces, list)
4554                 blk_freeze_queue_start(ns->queue);
4555         up_read(&ctrl->namespaces_rwsem);
4556 }
4557 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4558
4559 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4560 {
4561         struct nvme_ns *ns;
4562
4563         down_read(&ctrl->namespaces_rwsem);
4564         list_for_each_entry(ns, &ctrl->namespaces, list)
4565                 blk_mq_quiesce_queue(ns->queue);
4566         up_read(&ctrl->namespaces_rwsem);
4567 }
4568 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4569
4570 void nvme_start_queues(struct nvme_ctrl *ctrl)
4571 {
4572         struct nvme_ns *ns;
4573
4574         down_read(&ctrl->namespaces_rwsem);
4575         list_for_each_entry(ns, &ctrl->namespaces, list)
4576                 blk_mq_unquiesce_queue(ns->queue);
4577         up_read(&ctrl->namespaces_rwsem);
4578 }
4579 EXPORT_SYMBOL_GPL(nvme_start_queues);
4580
4581
4582 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4583 {
4584         struct nvme_ns *ns;
4585
4586         down_read(&ctrl->namespaces_rwsem);
4587         list_for_each_entry(ns, &ctrl->namespaces, list)
4588                 blk_sync_queue(ns->queue);
4589         up_read(&ctrl->namespaces_rwsem);
4590
4591         if (ctrl->admin_q)
4592                 blk_sync_queue(ctrl->admin_q);
4593 }
4594 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4595
4596 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4597 {
4598         if (file->f_op != &nvme_dev_fops)
4599                 return NULL;
4600         return file->private_data;
4601 }
4602 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
4603
4604 /*
4605  * Check we didn't inadvertently grow the command structure sizes:
4606  */
4607 static inline void _nvme_check_size(void)
4608 {
4609         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4610         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4611         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4612         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4613         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4614         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4615         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4616         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4617         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4618         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4619         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4620         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4621         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4622         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4623         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4624         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4625         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4626         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4627         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4628 }
4629
4630
4631 static int __init nvme_core_init(void)
4632 {
4633         int result = -ENOMEM;
4634
4635         _nvme_check_size();
4636
4637         nvme_wq = alloc_workqueue("nvme-wq",
4638                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4639         if (!nvme_wq)
4640                 goto out;
4641
4642         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4643                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4644         if (!nvme_reset_wq)
4645                 goto destroy_wq;
4646
4647         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4648                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4649         if (!nvme_delete_wq)
4650                 goto destroy_reset_wq;
4651
4652         result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4653         if (result < 0)
4654                 goto destroy_delete_wq;
4655
4656         nvme_class = class_create(THIS_MODULE, "nvme");
4657         if (IS_ERR(nvme_class)) {
4658                 result = PTR_ERR(nvme_class);
4659                 goto unregister_chrdev;
4660         }
4661         nvme_class->dev_uevent = nvme_class_uevent;
4662
4663         nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4664         if (IS_ERR(nvme_subsys_class)) {
4665                 result = PTR_ERR(nvme_subsys_class);
4666                 goto destroy_class;
4667         }
4668         return 0;
4669
4670 destroy_class:
4671         class_destroy(nvme_class);
4672 unregister_chrdev:
4673         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4674 destroy_delete_wq:
4675         destroy_workqueue(nvme_delete_wq);
4676 destroy_reset_wq:
4677         destroy_workqueue(nvme_reset_wq);
4678 destroy_wq:
4679         destroy_workqueue(nvme_wq);
4680 out:
4681         return result;
4682 }
4683
4684 static void __exit nvme_core_exit(void)
4685 {
4686         class_destroy(nvme_subsys_class);
4687         class_destroy(nvme_class);
4688         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4689         destroy_workqueue(nvme_delete_wq);
4690         destroy_workqueue(nvme_reset_wq);
4691         destroy_workqueue(nvme_wq);
4692         ida_destroy(&nvme_instance_ida);
4693 }
4694
4695 MODULE_LICENSE("GPL");
4696 MODULE_VERSION("1.0");
4697 module_init(nvme_core_init);
4698 module_exit(nvme_core_exit);