drivers/nvme/target/admin-cmd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe admin command implementation.
   4  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/module.h>
   8 #include <linux/rculist.h>
   9 #include <linux/part_stat.h>
  10
  11 #include <generated/utsrelease.h>
  12 #include <asm/unaligned.h>
  13 #include "nvmet.h"
  14
  15 u32 nvmet_get_log_page_len(struct nvme_command *cmd)
  16 {
  17         u32 len = le16_to_cpu(cmd->get_log_page.numdu);
  18
  19         len <<= 16;
  20         len += le16_to_cpu(cmd->get_log_page.numdl);
  21         /* NUMD is a 0's based value */
  22         len += 1;
  23         len *= sizeof(u32);
  24
  25         return len;
  26 }
  27
  28 static u32 nvmet_feat_data_len(struct nvmet_req *req, u32 cdw10)
  29 {
  30         switch (cdw10 & 0xff) {
  31         case NVME_FEAT_HOST_ID:
  32                 return sizeof(req->sq->ctrl->hostid);
  33         default:
  34                 return 0;
  35         }
  36 }
  37
  38 u64 nvmet_get_log_page_offset(struct nvme_command *cmd)
  39 {
  40         return le64_to_cpu(cmd->get_log_page.lpo);
  41 }
  42
  43 static void nvmet_execute_get_log_page_noop(struct nvmet_req *req)
  44 {
  45         nvmet_req_complete(req, nvmet_zero_sgl(req, 0, req->transfer_len));
  46 }
  47
  48 static void nvmet_execute_get_log_page_error(struct nvmet_req *req)
  49 {
  50         struct nvmet_ctrl *ctrl = req->sq->ctrl;
  51         unsigned long flags;
  52         off_t offset = 0;
  53         u64 slot;
  54         u64 i;
  55
  56         spin_lock_irqsave(&ctrl->error_lock, flags);
  57         slot = ctrl->err_counter % NVMET_ERROR_LOG_SLOTS;
  58
  59         for (i = 0; i < NVMET_ERROR_LOG_SLOTS; i++) {
  60                 if (nvmet_copy_to_sgl(req, offset, &ctrl->slots[slot],
  61                                 sizeof(struct nvme_error_slot)))
  62                         break;
  63
  64                 if (slot == 0)
  65                         slot = NVMET_ERROR_LOG_SLOTS - 1;
  66                 else
  67                         slot--;
  68                 offset += sizeof(struct nvme_error_slot);
  69         }
  70         spin_unlock_irqrestore(&ctrl->error_lock, flags);
  71         nvmet_req_complete(req, 0);
  72 }
  73
  74 static u16 nvmet_get_smart_log_nsid(struct nvmet_req *req,
  75                 struct nvme_smart_log *slog)
  76 {
  77         u64 host_reads, host_writes, data_units_read, data_units_written;
  78         u16 status;
  79
  80         status = nvmet_req_find_ns(req);
  81         if (status)
  82                 return status;
  83
  84         /* we don't have the right data for file backed ns */
  85         if (!req->ns->bdev)
  86                 return NVME_SC_SUCCESS;
  87
  88         host_reads = part_stat_read(req->ns->bdev, ios[READ]);
  89         data_units_read =
  90                 DIV_ROUND_UP(part_stat_read(req->ns->bdev, sectors[READ]), 1000);
  91         host_writes = part_stat_read(req->ns->bdev, ios[WRITE]);
  92         data_units_written =
  93                 DIV_ROUND_UP(part_stat_read(req->ns->bdev, sectors[WRITE]), 1000);
  94
  95         put_unaligned_le64(host_reads, &slog->host_reads[0]);
  96         put_unaligned_le64(data_units_read, &slog->data_units_read[0]);
  97         put_unaligned_le64(host_writes, &slog->host_writes[0]);
  98         put_unaligned_le64(data_units_written, &slog->data_units_written[0]);
  99
 100         return NVME_SC_SUCCESS;
 101 }
 102
 103 static u16 nvmet_get_smart_log_all(struct nvmet_req *req,
 104                 struct nvme_smart_log *slog)
 105 {
 106         u64 host_reads = 0, host_writes = 0;
 107         u64 data_units_read = 0, data_units_written = 0;
 108         struct nvmet_ns *ns;
 109         struct nvmet_ctrl *ctrl;
 110         unsigned long idx;
 111
 112         ctrl = req->sq->ctrl;
 113         xa_for_each(&ctrl->subsys->namespaces, idx, ns) {
 114                 /* we don't have the right data for file backed ns */
 115                 if (!ns->bdev)
 116                         continue;
 117                 host_reads += part_stat_read(ns->bdev, ios[READ]);
 118                 data_units_read += DIV_ROUND_UP(
 119                         part_stat_read(ns->bdev, sectors[READ]), 1000);
 120                 host_writes += part_stat_read(ns->bdev, ios[WRITE]);
 121                 data_units_written += DIV_ROUND_UP(
 122                         part_stat_read(ns->bdev, sectors[WRITE]), 1000);
 123         }
 124
 125         put_unaligned_le64(host_reads, &slog->host_reads[0]);
 126         put_unaligned_le64(data_units_read, &slog->data_units_read[0]);
 127         put_unaligned_le64(host_writes, &slog->host_writes[0]);
 128         put_unaligned_le64(data_units_written, &slog->data_units_written[0]);
 129
 130         return NVME_SC_SUCCESS;
 131 }
 132
 133 static void nvmet_execute_get_log_page_smart(struct nvmet_req *req)
 134 {
 135         struct nvme_smart_log *log;
 136         u16 status = NVME_SC_INTERNAL;
 137         unsigned long flags;
 138
 139         if (req->transfer_len != sizeof(*log))
 140                 goto out;
 141
 142         log = kzalloc(sizeof(*log), GFP_KERNEL);
 143         if (!log)
 144                 goto out;
 145
 146         if (req->cmd->get_log_page.nsid == cpu_to_le32(NVME_NSID_ALL))
 147                 status = nvmet_get_smart_log_all(req, log);
 148         else
 149                 status = nvmet_get_smart_log_nsid(req, log);
 150         if (status)
 151                 goto out_free_log;
 152
 153         spin_lock_irqsave(&req->sq->ctrl->error_lock, flags);
 154         put_unaligned_le64(req->sq->ctrl->err_counter,
 155                         &log->num_err_log_entries);
 156         spin_unlock_irqrestore(&req->sq->ctrl->error_lock, flags);
 157
 158         status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
 159 out_free_log:
 160         kfree(log);
 161 out:
 162         nvmet_req_complete(req, status);
 163 }
 164
 165 static void nvmet_execute_get_log_cmd_effects_ns(struct nvmet_req *req)
 166 {
 167         u16 status = NVME_SC_INTERNAL;
 168         struct nvme_effects_log *log;
 169
 170         log = kzalloc(sizeof(*log), GFP_KERNEL);
 171         if (!log)
 172                 goto out;
 173
 174         log->acs[nvme_admin_get_log_page]       = cpu_to_le32(1 << 0);
 175         log->acs[nvme_admin_identify]           = cpu_to_le32(1 << 0);
 176         log->acs[nvme_admin_abort_cmd]          = cpu_to_le32(1 << 0);
 177         log->acs[nvme_admin_set_features]       = cpu_to_le32(1 << 0);
 178         log->acs[nvme_admin_get_features]       = cpu_to_le32(1 << 0);
 179         log->acs[nvme_admin_async_event]        = cpu_to_le32(1 << 0);
 180         log->acs[nvme_admin_keep_alive]         = cpu_to_le32(1 << 0);
 181
 182         log->iocs[nvme_cmd_read]                = cpu_to_le32(1 << 0);
 183         log->iocs[nvme_cmd_write]               = cpu_to_le32(1 << 0);
 184         log->iocs[nvme_cmd_flush]               = cpu_to_le32(1 << 0);
 185         log->iocs[nvme_cmd_dsm]                 = cpu_to_le32(1 << 0);
 186         log->iocs[nvme_cmd_write_zeroes]        = cpu_to_le32(1 << 0);
 187
 188         status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
 189
 190         kfree(log);
 191 out:
 192         nvmet_req_complete(req, status);
 193 }
 194
 195 static void nvmet_execute_get_log_changed_ns(struct nvmet_req *req)
 196 {
 197         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 198         u16 status = NVME_SC_INTERNAL;
 199         size_t len;
 200
 201         if (req->transfer_len != NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32))
 202                 goto out;
 203
 204         mutex_lock(&ctrl->lock);
 205         if (ctrl->nr_changed_ns == U32_MAX)
 206                 len = sizeof(__le32);
 207         else
 208                 len = ctrl->nr_changed_ns * sizeof(__le32);
 209         status = nvmet_copy_to_sgl(req, 0, ctrl->changed_ns_list, len);
 210         if (!status)
 211                 status = nvmet_zero_sgl(req, len, req->transfer_len - len);
 212         ctrl->nr_changed_ns = 0;
 213         nvmet_clear_aen_bit(req, NVME_AEN_BIT_NS_ATTR);
 214         mutex_unlock(&ctrl->lock);
 215 out:
 216         nvmet_req_complete(req, status);
 217 }
 218
 219 static u32 nvmet_format_ana_group(struct nvmet_req *req, u32 grpid,
 220                 struct nvme_ana_group_desc *desc)
 221 {
 222         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 223         struct nvmet_ns *ns;
 224         unsigned long idx;
 225         u32 count = 0;
 226
 227         if (!(req->cmd->get_log_page.lsp & NVME_ANA_LOG_RGO)) {
 228                 xa_for_each(&ctrl->subsys->namespaces, idx, ns)
 229                         if (ns->anagrpid == grpid)
 230                                 desc->nsids[count++] = cpu_to_le32(ns->nsid);
 231         }
 232
 233         desc->grpid = cpu_to_le32(grpid);
 234         desc->nnsids = cpu_to_le32(count);
 235         desc->chgcnt = cpu_to_le64(nvmet_ana_chgcnt);
 236         desc->state = req->port->ana_state[grpid];
 237         memset(desc->rsvd17, 0, sizeof(desc->rsvd17));
 238         return sizeof(struct nvme_ana_group_desc) + count * sizeof(__le32);
 239 }
 240
 241 static void nvmet_execute_get_log_page_ana(struct nvmet_req *req)
 242 {
 243         struct nvme_ana_rsp_hdr hdr = { 0, };
 244         struct nvme_ana_group_desc *desc;
 245         size_t offset = sizeof(struct nvme_ana_rsp_hdr); /* start beyond hdr */
 246         size_t len;
 247         u32 grpid;
 248         u16 ngrps = 0;
 249         u16 status;
 250
 251         status = NVME_SC_INTERNAL;
 252         desc = kmalloc(sizeof(struct nvme_ana_group_desc) +
 253                         NVMET_MAX_NAMESPACES * sizeof(__le32), GFP_KERNEL);
 254         if (!desc)
 255                 goto out;
 256
 257         down_read(&nvmet_ana_sem);
 258         for (grpid = 1; grpid <= NVMET_MAX_ANAGRPS; grpid++) {
 259                 if (!nvmet_ana_group_enabled[grpid])
 260                         continue;
 261                 len = nvmet_format_ana_group(req, grpid, desc);
 262                 status = nvmet_copy_to_sgl(req, offset, desc, len);
 263                 if (status)
 264                         break;
 265                 offset += len;
 266                 ngrps++;
 267         }
 268         for ( ; grpid <= NVMET_MAX_ANAGRPS; grpid++) {
 269                 if (nvmet_ana_group_enabled[grpid])
 270                         ngrps++;
 271         }
 272
 273         hdr.chgcnt = cpu_to_le64(nvmet_ana_chgcnt);
 274         hdr.ngrps = cpu_to_le16(ngrps);
 275         nvmet_clear_aen_bit(req, NVME_AEN_BIT_ANA_CHANGE);
 276         up_read(&nvmet_ana_sem);
 277
 278         kfree(desc);
 279
 280         /* copy the header last once we know the number of groups */
 281         status = nvmet_copy_to_sgl(req, 0, &hdr, sizeof(hdr));
 282 out:
 283         nvmet_req_complete(req, status);
 284 }
 285
 286 static void nvmet_execute_get_log_page(struct nvmet_req *req)
 287 {
 288         if (!nvmet_check_transfer_len(req, nvmet_get_log_page_len(req->cmd)))
 289                 return;
 290
 291         switch (req->cmd->get_log_page.lid) {
 292         case NVME_LOG_ERROR:
 293                 return nvmet_execute_get_log_page_error(req);
 294         case NVME_LOG_SMART:
 295                 return nvmet_execute_get_log_page_smart(req);
 296         case NVME_LOG_FW_SLOT:
 297                 /*
 298                  * We only support a single firmware slot which always is
 299                  * active, so we can zero out the whole firmware slot log and
 300                  * still claim to fully implement this mandatory log page.
 301                  */
 302                 return nvmet_execute_get_log_page_noop(req);
 303         case NVME_LOG_CHANGED_NS:
 304                 return nvmet_execute_get_log_changed_ns(req);
 305         case NVME_LOG_CMD_EFFECTS:
 306                 return nvmet_execute_get_log_cmd_effects_ns(req);
 307         case NVME_LOG_ANA:
 308                 return nvmet_execute_get_log_page_ana(req);
 309         }
 310         pr_err("unhandled lid %d on qid %d\n",
 311                req->cmd->get_log_page.lid, req->sq->qid);
 312         req->error_loc = offsetof(struct nvme_get_log_page_command, lid);
 313         nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR);
 314 }
 315
 316 static void nvmet_id_set_model_number(struct nvme_id_ctrl *id,
 317                                       struct nvmet_subsys *subsys)
 318 {
 319         const char *model = NVMET_DEFAULT_CTRL_MODEL;
 320         struct nvmet_subsys_model *subsys_model;
 321
 322         rcu_read_lock();
 323         subsys_model = rcu_dereference(subsys->model);
 324         if (subsys_model)
 325                 model = subsys_model->number;
 326         memcpy_and_pad(id->mn, sizeof(id->mn), model, strlen(model), ' ');
 327         rcu_read_unlock();
 328 }
 329
 330 static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 331 {
 332         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 333         struct nvme_id_ctrl *id;
 334         u32 cmd_capsule_size;
 335         u16 status = 0;
 336
 337         id = kzalloc(sizeof(*id), GFP_KERNEL);
 338         if (!id) {
 339                 status = NVME_SC_INTERNAL;
 340                 goto out;
 341         }
 342
 343         /* XXX: figure out how to assign real vendors IDs. */
 344         id->vid = 0;
 345         id->ssvid = 0;
 346
 347         memset(id->sn, ' ', sizeof(id->sn));
 348         bin2hex(id->sn, &ctrl->subsys->serial,
 349                 min(sizeof(ctrl->subsys->serial), sizeof(id->sn) / 2));
 350         nvmet_id_set_model_number(id, ctrl->subsys);
 351         memcpy_and_pad(id->fr, sizeof(id->fr),
 352                        UTS_RELEASE, strlen(UTS_RELEASE), ' ');
 353
 354         id->rab = 6;
 355
 356         /*
 357          * XXX: figure out how we can assign a IEEE OUI, but until then
 358          * the safest is to leave it as zeroes.
 359          */
 360
 361         /* we support multiple ports, multiples hosts and ANA: */
 362         id->cmic = (1 << 0) | (1 << 1) | (1 << 3);
 363
 364         /* Limit MDTS according to transport capability */
 365         if (ctrl->ops->get_mdts)
 366                 id->mdts = ctrl->ops->get_mdts(ctrl);
 367         else
 368                 id->mdts = 0;
 369
 370         id->cntlid = cpu_to_le16(ctrl->cntlid);
 371         id->ver = cpu_to_le32(ctrl->subsys->ver);
 372
 373         /* XXX: figure out what to do about RTD3R/RTD3 */
 374         id->oaes = cpu_to_le32(NVMET_AEN_CFG_OPTIONAL);
 375         id->ctratt = cpu_to_le32(NVME_CTRL_ATTR_HID_128_BIT |
 376                 NVME_CTRL_ATTR_TBKAS);
 377
 378         id->oacs = 0;
 379
 380         /*
 381          * We don't really have a practical limit on the number of abort
 382          * comands.  But we don't do anything useful for abort either, so
 383          * no point in allowing more abort commands than the spec requires.
 384          */
 385         id->acl = 3;
 386
 387         id->aerl = NVMET_ASYNC_EVENTS - 1;
 388
 389         /* first slot is read-only, only one slot supported */
 390         id->frmw = (1 << 0) | (1 << 1);
 391         id->lpa = (1 << 0) | (1 << 1) | (1 << 2);
 392         id->elpe = NVMET_ERROR_LOG_SLOTS - 1;
 393         id->npss = 0;
 394
 395         /* We support keep-alive timeout in granularity of seconds */
 396         id->kas = cpu_to_le16(NVMET_KAS);
 397
 398         id->sqes = (0x6 << 4) | 0x6;
 399         id->cqes = (0x4 << 4) | 0x4;
 400
 401         /* no enforcement soft-limit for maxcmd - pick arbitrary high value */
 402         id->maxcmd = cpu_to_le16(NVMET_MAX_CMD);
 403
 404         id->nn = cpu_to_le32(ctrl->subsys->max_nsid);
 405         id->mnan = cpu_to_le32(NVMET_MAX_NAMESPACES);
 406         id->oncs = cpu_to_le16(NVME_CTRL_ONCS_DSM |
 407                         NVME_CTRL_ONCS_WRITE_ZEROES);
 408
 409         /* XXX: don't report vwc if the underlying device is write through */
 410         id->vwc = NVME_CTRL_VWC_PRESENT;
 411
 412         /*
 413          * We can't support atomic writes bigger than a LBA without support
 414          * from the backend device.
 415          */
 416         id->awun = 0;
 417         id->awupf = 0;
 418
 419         id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */
 420         if (ctrl->ops->flags & NVMF_KEYED_SGLS)
 421                 id->sgls |= cpu_to_le32(1 << 2);
 422         if (req->port->inline_data_size)
 423                 id->sgls |= cpu_to_le32(1 << 20);
 424
 425         strlcpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn));
 426
 427         /*
 428          * Max command capsule size is sqe + in-capsule data size.
 429          * Disable in-capsule data for Metadata capable controllers.
 430          */
 431         cmd_capsule_size = sizeof(struct nvme_command);
 432         if (!ctrl->pi_support)
 433                 cmd_capsule_size += req->port->inline_data_size;
 434         id->ioccsz = cpu_to_le32(cmd_capsule_size / 16);
 435
 436         /* Max response capsule size is cqe */
 437         id->iorcsz = cpu_to_le32(sizeof(struct nvme_completion) / 16);
 438
 439         id->msdbd = ctrl->ops->msdbd;
 440
 441         id->anacap = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4);
 442         id->anatt = 10; /* random value */
 443         id->anagrpmax = cpu_to_le32(NVMET_MAX_ANAGRPS);
 444         id->nanagrpid = cpu_to_le32(NVMET_MAX_ANAGRPS);
 445
 446         /*
 447          * Meh, we don't really support any power state.  Fake up the same
 448          * values that qemu does.
 449          */
 450         id->psd[0].max_power = cpu_to_le16(0x9c4);
 451         id->psd[0].entry_lat = cpu_to_le32(0x10);
 452         id->psd[0].exit_lat = cpu_to_le32(0x4);
 453
 454         id->nwpc = 1 << 0; /* write protect and no write protect */
 455
 456         status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));
 457
 458         kfree(id);
 459 out:
 460         nvmet_req_complete(req, status);
 461 }
 462
 463 static void nvmet_execute_identify_ns(struct nvmet_req *req)
 464 {
 465         struct nvme_id_ns *id;
 466         u16 status;
 467
 468         if (le32_to_cpu(req->cmd->identify.nsid) == NVME_NSID_ALL) {
 469                 req->error_loc = offsetof(struct nvme_identify, nsid);
 470                 status = NVME_SC_INVALID_NS | NVME_SC_DNR;
 471                 goto out;
 472         }
 473
 474         id = kzalloc(sizeof(*id), GFP_KERNEL);
 475         if (!id) {
 476                 status = NVME_SC_INTERNAL;
 477                 goto out;
 478         }
 479
 480         /* return an all zeroed buffer if we can't find an active namespace */
 481         status = nvmet_req_find_ns(req);
 482         if (status) {
 483                 status = 0;
 484                 goto done;
 485         }
 486
 487         nvmet_ns_revalidate(req->ns);
 488
 489         /*
 490          * nuse = ncap = nsze isn't always true, but we have no way to find
 491          * that out from the underlying device.
 492          */
 493         id->ncap = id->nsze =
 494                 cpu_to_le64(req->ns->size >> req->ns->blksize_shift);
 495         switch (req->port->ana_state[req->ns->anagrpid]) {
 496         case NVME_ANA_INACCESSIBLE:
 497         case NVME_ANA_PERSISTENT_LOSS:
 498                 break;
 499         default:
 500                 id->nuse = id->nsze;
 501                 break;
 502         }
 503
 504         if (req->ns->bdev)
 505                 nvmet_bdev_set_limits(req->ns->bdev, id);
 506
 507         /*
 508          * We just provide a single LBA format that matches what the
 509          * underlying device reports.
 510          */
 511         id->nlbaf = 0;
 512         id->flbas = 0;
 513
 514         /*
 515          * Our namespace might always be shared.  Not just with other
 516          * controllers, but also with any other user of the block device.
 517          */
 518         id->nmic = (1 << 0);
 519         id->anagrpid = cpu_to_le32(req->ns->anagrpid);
 520
 521         memcpy(&id->nguid, &req->ns->nguid, sizeof(id->nguid));
 522
 523         id->lbaf[0].ds = req->ns->blksize_shift;
 524
 525         if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns)) {
 526                 id->dpc = NVME_NS_DPC_PI_FIRST | NVME_NS_DPC_PI_LAST |
 527                           NVME_NS_DPC_PI_TYPE1 | NVME_NS_DPC_PI_TYPE2 |
 528                           NVME_NS_DPC_PI_TYPE3;
 529                 id->mc = NVME_MC_EXTENDED_LBA;
 530                 id->dps = req->ns->pi_type;
 531                 id->flbas = NVME_NS_FLBAS_META_EXT;
 532                 id->lbaf[0].ms = cpu_to_le16(req->ns->metadata_size);
 533         }
 534
 535         if (req->ns->readonly)
 536                 id->nsattr |= (1 << 0);
 537 done:
 538         if (!status)
 539                 status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));
 540
 541         kfree(id);
 542 out:
 543         nvmet_req_complete(req, status);
 544 }
 545
 546 static void nvmet_execute_identify_nslist(struct nvmet_req *req)
 547 {
 548         static const int buf_size = NVME_IDENTIFY_DATA_SIZE;
 549         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 550         struct nvmet_ns *ns;
 551         unsigned long idx;
 552         u32 min_nsid = le32_to_cpu(req->cmd->identify.nsid);
 553         __le32 *list;
 554         u16 status = 0;
 555         int i = 0;
 556
 557         list = kzalloc(buf_size, GFP_KERNEL);
 558         if (!list) {
 559                 status = NVME_SC_INTERNAL;
 560                 goto out;
 561         }
 562
 563         xa_for_each(&ctrl->subsys->namespaces, idx, ns) {
 564                 if (ns->nsid <= min_nsid)
 565                         continue;
 566                 list[i++] = cpu_to_le32(ns->nsid);
 567                 if (i == buf_size / sizeof(__le32))
 568                         break;
 569         }
 570
 571         status = nvmet_copy_to_sgl(req, 0, list, buf_size);
 572
 573         kfree(list);
 574 out:
 575         nvmet_req_complete(req, status);
 576 }
 577
 578 static u16 nvmet_copy_ns_identifier(struct nvmet_req *req, u8 type, u8 len,
 579                                     void *id, off_t *off)
 580 {
 581         struct nvme_ns_id_desc desc = {
 582                 .nidt = type,
 583                 .nidl = len,
 584         };
 585         u16 status;
 586
 587         status = nvmet_copy_to_sgl(req, *off, &desc, sizeof(desc));
 588         if (status)
 589                 return status;
 590         *off += sizeof(desc);
 591
 592         status = nvmet_copy_to_sgl(req, *off, id, len);
 593         if (status)
 594                 return status;
 595         *off += len;
 596
 597         return 0;
 598 }
 599
 600 static void nvmet_execute_identify_desclist(struct nvmet_req *req)
 601 {
 602         off_t off = 0;
 603         u16 status;
 604
 605         status = nvmet_req_find_ns(req);
 606         if (status)
 607                 goto out;
 608
 609         if (memchr_inv(&req->ns->uuid, 0, sizeof(req->ns->uuid))) {
 610                 status = nvmet_copy_ns_identifier(req, NVME_NIDT_UUID,
 611                                                   NVME_NIDT_UUID_LEN,
 612                                                   &req->ns->uuid, &off);
 613                 if (status)
 614                         goto out;
 615         }
 616         if (memchr_inv(req->ns->nguid, 0, sizeof(req->ns->nguid))) {
 617                 status = nvmet_copy_ns_identifier(req, NVME_NIDT_NGUID,
 618                                                   NVME_NIDT_NGUID_LEN,
 619                                                   &req->ns->nguid, &off);
 620                 if (status)
 621                         goto out;
 622         }
 623
 624         if (sg_zero_buffer(req->sg, req->sg_cnt, NVME_IDENTIFY_DATA_SIZE - off,
 625                         off) != NVME_IDENTIFY_DATA_SIZE - off)
 626                 status = NVME_SC_INTERNAL | NVME_SC_DNR;
 627
 628 out:
 629         nvmet_req_complete(req, status);
 630 }
 631
 632 static void nvmet_execute_identify(struct nvmet_req *req)
 633 {
 634         if (!nvmet_check_transfer_len(req, NVME_IDENTIFY_DATA_SIZE))
 635                 return;
 636
 637         switch (req->cmd->identify.cns) {
 638         case NVME_ID_CNS_NS:
 639                 return nvmet_execute_identify_ns(req);
 640         case NVME_ID_CNS_CTRL:
 641                 return nvmet_execute_identify_ctrl(req);
 642         case NVME_ID_CNS_NS_ACTIVE_LIST:
 643                 return nvmet_execute_identify_nslist(req);
 644         case NVME_ID_CNS_NS_DESC_LIST:
 645                 return nvmet_execute_identify_desclist(req);
 646         }
 647
 648         pr_err("unhandled identify cns %d on qid %d\n",
 649                req->cmd->identify.cns, req->sq->qid);
 650         req->error_loc = offsetof(struct nvme_identify, cns);
 651         nvmet_req_complete(req, NVME_SC_INVALID_FIELD | NVME_SC_DNR);
 652 }
 653
 654 /*
 655  * A "minimum viable" abort implementation: the command is mandatory in the
 656  * spec, but we are not required to do any useful work.  We couldn't really
 657  * do a useful abort, so don't bother even with waiting for the command
 658  * to be exectuted and return immediately telling the command to abort
 659  * wasn't found.
 660  */
 661 static void nvmet_execute_abort(struct nvmet_req *req)
 662 {
 663         if (!nvmet_check_transfer_len(req, 0))
 664                 return;
 665         nvmet_set_result(req, 1);
 666         nvmet_req_complete(req, 0);
 667 }
 668
 669 static u16 nvmet_write_protect_flush_sync(struct nvmet_req *req)
 670 {
 671         u16 status;
 672
 673         if (req->ns->file)
 674                 status = nvmet_file_flush(req);
 675         else
 676                 status = nvmet_bdev_flush(req);
 677
 678         if (status)
 679                 pr_err("write protect flush failed nsid: %u\n", req->ns->nsid);
 680         return status;
 681 }
 682
 683 static u16 nvmet_set_feat_write_protect(struct nvmet_req *req)
 684 {
 685         u32 write_protect = le32_to_cpu(req->cmd->common.cdw11);
 686         struct nvmet_subsys *subsys = nvmet_req_subsys(req);
 687         u16 status;
 688
 689         status = nvmet_req_find_ns(req);
 690         if (status)
 691                 return status;
 692
 693         mutex_lock(&subsys->lock);
 694         switch (write_protect) {
 695         case NVME_NS_WRITE_PROTECT:
 696                 req->ns->readonly = true;
 697                 status = nvmet_write_protect_flush_sync(req);
 698                 if (status)
 699                         req->ns->readonly = false;
 700                 break;
 701         case NVME_NS_NO_WRITE_PROTECT:
 702                 req->ns->readonly = false;
 703                 status = 0;
 704                 break;
 705         default:
 706                 break;
 707         }
 708
 709         if (!status)
 710                 nvmet_ns_changed(subsys, req->ns->nsid);
 711         mutex_unlock(&subsys->lock);
 712         return status;
 713 }
 714
 715 u16 nvmet_set_feat_kato(struct nvmet_req *req)
 716 {
 717         u32 val32 = le32_to_cpu(req->cmd->common.cdw11);
 718
 719         nvmet_stop_keep_alive_timer(req->sq->ctrl);
 720         req->sq->ctrl->kato = DIV_ROUND_UP(val32, 1000);
 721         nvmet_start_keep_alive_timer(req->sq->ctrl);
 722
 723         nvmet_set_result(req, req->sq->ctrl->kato);
 724
 725         return 0;
 726 }
 727
 728 u16 nvmet_set_feat_async_event(struct nvmet_req *req, u32 mask)
 729 {
 730         u32 val32 = le32_to_cpu(req->cmd->common.cdw11);
 731
 732         if (val32 & ~mask) {
 733                 req->error_loc = offsetof(struct nvme_common_command, cdw11);
 734                 return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 735         }
 736
 737         WRITE_ONCE(req->sq->ctrl->aen_enabled, val32);
 738         nvmet_set_result(req, val32);
 739
 740         return 0;
 741 }
 742
 743 void nvmet_execute_set_features(struct nvmet_req *req)
 744 {
 745         struct nvmet_subsys *subsys = nvmet_req_subsys(req);
 746         u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
 747         u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11);
 748         u16 status = 0;
 749         u16 nsqr;
 750         u16 ncqr;
 751
 752         if (!nvmet_check_transfer_len(req, 0))
 753                 return;
 754
 755         switch (cdw10 & 0xff) {
 756         case NVME_FEAT_NUM_QUEUES:
 757                 ncqr = (cdw11 >> 16) & 0xffff;
 758                 nsqr = cdw11 & 0xffff;
 759                 if (ncqr == 0xffff || nsqr == 0xffff) {
 760                         status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 761                         break;
 762                 }
 763                 nvmet_set_result(req,
 764                         (subsys->max_qid - 1) | ((subsys->max_qid - 1) << 16));
 765                 break;
 766         case NVME_FEAT_KATO:
 767                 status = nvmet_set_feat_kato(req);
 768                 break;
 769         case NVME_FEAT_ASYNC_EVENT:
 770                 status = nvmet_set_feat_async_event(req, NVMET_AEN_CFG_ALL);
 771                 break;
 772         case NVME_FEAT_HOST_ID:
 773                 status = NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
 774                 break;
 775         case NVME_FEAT_WRITE_PROTECT:
 776                 status = nvmet_set_feat_write_protect(req);
 777                 break;
 778         default:
 779                 req->error_loc = offsetof(struct nvme_common_command, cdw10);
 780                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 781                 break;
 782         }
 783
 784         nvmet_req_complete(req, status);
 785 }
 786
 787 static u16 nvmet_get_feat_write_protect(struct nvmet_req *req)
 788 {
 789         struct nvmet_subsys *subsys = nvmet_req_subsys(req);
 790         u32 result;
 791
 792         result = nvmet_req_find_ns(req);
 793         if (result)
 794                 return result;
 795
 796         mutex_lock(&subsys->lock);
 797         if (req->ns->readonly == true)
 798                 result = NVME_NS_WRITE_PROTECT;
 799         else
 800                 result = NVME_NS_NO_WRITE_PROTECT;
 801         nvmet_set_result(req, result);
 802         mutex_unlock(&subsys->lock);
 803
 804         return 0;
 805 }
 806
 807 void nvmet_get_feat_kato(struct nvmet_req *req)
 808 {
 809         nvmet_set_result(req, req->sq->ctrl->kato * 1000);
 810 }
 811
 812 void nvmet_get_feat_async_event(struct nvmet_req *req)
 813 {
 814         nvmet_set_result(req, READ_ONCE(req->sq->ctrl->aen_enabled));
 815 }
 816
 817 void nvmet_execute_get_features(struct nvmet_req *req)
 818 {
 819         struct nvmet_subsys *subsys = nvmet_req_subsys(req);
 820         u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
 821         u16 status = 0;
 822
 823         if (!nvmet_check_transfer_len(req, nvmet_feat_data_len(req, cdw10)))
 824                 return;
 825
 826         switch (cdw10 & 0xff) {
 827         /*
 828          * These features are mandatory in the spec, but we don't
 829          * have a useful way to implement them.  We'll eventually
 830          * need to come up with some fake values for these.
 831          */
 832 #if 0
 833         case NVME_FEAT_ARBITRATION:
 834                 break;
 835         case NVME_FEAT_POWER_MGMT:
 836                 break;
 837         case NVME_FEAT_TEMP_THRESH:
 838                 break;
 839         case NVME_FEAT_ERR_RECOVERY:
 840                 break;
 841         case NVME_FEAT_IRQ_COALESCE:
 842                 break;
 843         case NVME_FEAT_IRQ_CONFIG:
 844                 break;
 845         case NVME_FEAT_WRITE_ATOMIC:
 846                 break;
 847 #endif
 848         case NVME_FEAT_ASYNC_EVENT:
 849                 nvmet_get_feat_async_event(req);
 850                 break;
 851         case NVME_FEAT_VOLATILE_WC:
 852                 nvmet_set_result(req, 1);
 853                 break;
 854         case NVME_FEAT_NUM_QUEUES:
 855                 nvmet_set_result(req,
 856                         (subsys->max_qid-1) | ((subsys->max_qid-1) << 16));
 857                 break;
 858         case NVME_FEAT_KATO:
 859                 nvmet_get_feat_kato(req);
 860                 break;
 861         case NVME_FEAT_HOST_ID:
 862                 /* need 128-bit host identifier flag */
 863                 if (!(req->cmd->common.cdw11 & cpu_to_le32(1 << 0))) {
 864                         req->error_loc =
 865                                 offsetof(struct nvme_common_command, cdw11);
 866                         status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 867                         break;
 868                 }
 869
 870                 status = nvmet_copy_to_sgl(req, 0, &req->sq->ctrl->hostid,
 871                                 sizeof(req->sq->ctrl->hostid));
 872                 break;
 873         case NVME_FEAT_WRITE_PROTECT:
 874                 status = nvmet_get_feat_write_protect(req);
 875                 break;
 876         default:
 877                 req->error_loc =
 878                         offsetof(struct nvme_common_command, cdw10);
 879                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 880                 break;
 881         }
 882
 883         nvmet_req_complete(req, status);
 884 }
 885
 886 void nvmet_execute_async_event(struct nvmet_req *req)
 887 {
 888         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 889
 890         if (!nvmet_check_transfer_len(req, 0))
 891                 return;
 892
 893         mutex_lock(&ctrl->lock);
 894         if (ctrl->nr_async_event_cmds >= NVMET_ASYNC_EVENTS) {
 895                 mutex_unlock(&ctrl->lock);
 896                 nvmet_req_complete(req, NVME_SC_ASYNC_LIMIT | NVME_SC_DNR);
 897                 return;
 898         }
 899         ctrl->async_event_cmds[ctrl->nr_async_event_cmds++] = req;
 900         mutex_unlock(&ctrl->lock);
 901
 902         schedule_work(&ctrl->async_event_work);
 903 }
 904
 905 void nvmet_execute_keep_alive(struct nvmet_req *req)
 906 {
 907         struct nvmet_ctrl *ctrl = req->sq->ctrl;
 908
 909         if (!nvmet_check_transfer_len(req, 0))
 910                 return;
 911
 912         pr_debug("ctrl %d update keep-alive timer for %d secs\n",
 913                 ctrl->cntlid, ctrl->kato);
 914
 915         mod_delayed_work(system_wq, &ctrl->ka_work, ctrl->kato * HZ);
 916         nvmet_req_complete(req, 0);
 917 }
 918
 919 u16 nvmet_parse_admin_cmd(struct nvmet_req *req)
 920 {
 921         struct nvme_command *cmd = req->cmd;
 922         u16 ret;
 923
 924         if (nvme_is_fabrics(cmd))
 925                 return nvmet_parse_fabrics_cmd(req);
 926         if (nvmet_req_subsys(req)->type == NVME_NQN_DISC)
 927                 return nvmet_parse_discovery_cmd(req);
 928
 929         ret = nvmet_check_ctrl_status(req, cmd);
 930         if (unlikely(ret))
 931                 return ret;
 932
 933         if (nvmet_req_passthru_ctrl(req))
 934                 return nvmet_parse_passthru_admin_cmd(req);
 935
 936         switch (cmd->common.opcode) {
 937         case nvme_admin_get_log_page:
 938                 req->execute = nvmet_execute_get_log_page;
 939                 return 0;
 940         case nvme_admin_identify:
 941                 req->execute = nvmet_execute_identify;
 942                 return 0;
 943         case nvme_admin_abort_cmd:
 944                 req->execute = nvmet_execute_abort;
 945                 return 0;
 946         case nvme_admin_set_features:
 947                 req->execute = nvmet_execute_set_features;
 948                 return 0;
 949         case nvme_admin_get_features:
 950                 req->execute = nvmet_execute_get_features;
 951                 return 0;
 952         case nvme_admin_async_event:
 953                 req->execute = nvmet_execute_async_event;
 954                 return 0;
 955         case nvme_admin_keep_alive:
 956                 req->execute = nvmet_execute_keep_alive;
 957                 return 0;
 958         }
 959
 960         pr_err("unhandled cmd %d on qid %d\n", cmd->common.opcode,
 961                req->sq->qid);
 962         req->error_loc = offsetof(struct nvme_common_command, opcode);
 963         return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
 964 }