drivers/scsi/lpfc/lpfc_nvme.c

   1 /*******************************************************************
   2  * This file is part of the Emulex Linux Device Driver for         *
   3  * Fibre Channel Host Bus Adapters.                                *
   4  * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
   5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
   6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
   7  * EMULEX and SLI are trademarks of Emulex.                        *
   8  * www.broadcom.com                                                *
   9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
  10  *                                                                 *
  11  * This program is free software; you can redistribute it and/or   *
  12  * modify it under the terms of version 2 of the GNU General       *
  13  * Public License as published by the Free Software Foundation.    *
  14  * This program is distributed in the hope that it will be useful. *
  15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  20  * more details, a copy of which can be found in the file COPYING  *
  21  * included with this package.                                     *
  22  ********************************************************************/
  23 #include <linux/pci.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/delay.h>
  27 #include <asm/unaligned.h>
  28 #include <linux/crc-t10dif.h>
  29 #include <net/checksum.h>
  30
  31 #include <scsi/scsi.h>
  32 #include <scsi/scsi_device.h>
  33 #include <scsi/scsi_eh.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi_tcq.h>
  36 #include <scsi/scsi_transport_fc.h>
  37 #include <scsi/fc/fc_fs.h>
  38
  39 #include "lpfc_version.h"
  40 #include "lpfc_hw4.h"
  41 #include "lpfc_hw.h"
  42 #include "lpfc_sli.h"
  43 #include "lpfc_sli4.h"
  44 #include "lpfc_nl.h"
  45 #include "lpfc_disc.h"
  46 #include "lpfc.h"
  47 #include "lpfc_nvme.h"
  48 #include "lpfc_scsi.h"
  49 #include "lpfc_logmsg.h"
  50 #include "lpfc_crtn.h"
  51 #include "lpfc_vport.h"
  52 #include "lpfc_debugfs.h"
  53
  54 /* NVME initiator-based functions */
  55
  56 static struct lpfc_io_buf *
  57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
  58                   int idx, int expedite);
  59
  60 static void
  61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
  62
  63 static struct nvme_fc_port_template lpfc_nvme_template;
  64
  65 static union lpfc_wqe128 lpfc_iread_cmd_template;
  66 static union lpfc_wqe128 lpfc_iwrite_cmd_template;
  67 static union lpfc_wqe128 lpfc_icmnd_cmd_template;
  68
  69 /* Setup WQE templates for NVME IOs */
  70 void
  71 lpfc_nvme_cmd_template(void)
  72 {
  73         union lpfc_wqe128 *wqe;
  74
  75         /* IREAD template */
  76         wqe = &lpfc_iread_cmd_template;
  77         memset(wqe, 0, sizeof(union lpfc_wqe128));
  78
  79         /* Word 0, 1, 2 - BDE is variable */
  80
  81         /* Word 3 - cmd_buff_len, payload_offset_len is zero */
  82
  83         /* Word 4 - total_xfer_len is variable */
  84
  85         /* Word 5 - is zero */
  86
  87         /* Word 6 - ctxt_tag, xri_tag is variable */
  88
  89         /* Word 7 */
  90         bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
  91         bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
  92         bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
  93         bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
  94
  95         /* Word 8 - abort_tag is variable */
  96
  97         /* Word 9  - reqtag is variable */
  98
  99         /* Word 10 - dbde, wqes is variable */
 100         bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
 101         bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1);
 102         bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
 103         bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
 104         bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
 105         bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
 106
 107         /* Word 11 - pbde is variable */
 108         bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD);
 109         bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 110         bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
 111
 112         /* Word 12 - is zero */
 113
 114         /* Word 13, 14, 15 - PBDE is variable */
 115
 116         /* IWRITE template */
 117         wqe = &lpfc_iwrite_cmd_template;
 118         memset(wqe, 0, sizeof(union lpfc_wqe128));
 119
 120         /* Word 0, 1, 2 - BDE is variable */
 121
 122         /* Word 3 - cmd_buff_len, payload_offset_len is zero */
 123
 124         /* Word 4 - total_xfer_len is variable */
 125
 126         /* Word 5 - initial_xfer_len is variable */
 127
 128         /* Word 6 - ctxt_tag, xri_tag is variable */
 129
 130         /* Word 7 */
 131         bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
 132         bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
 133         bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
 134         bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
 135
 136         /* Word 8 - abort_tag is variable */
 137
 138         /* Word 9  - reqtag is variable */
 139
 140         /* Word 10 - dbde, wqes is variable */
 141         bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
 142         bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1);
 143         bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
 144         bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
 145         bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
 146         bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
 147
 148         /* Word 11 - pbde is variable */
 149         bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD);
 150         bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 151         bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
 152
 153         /* Word 12 - is zero */
 154
 155         /* Word 13, 14, 15 - PBDE is variable */
 156
 157         /* ICMND template */
 158         wqe = &lpfc_icmnd_cmd_template;
 159         memset(wqe, 0, sizeof(union lpfc_wqe128));
 160
 161         /* Word 0, 1, 2 - BDE is variable */
 162
 163         /* Word 3 - payload_offset_len is variable */
 164
 165         /* Word 4, 5 - is zero */
 166
 167         /* Word 6 - ctxt_tag, xri_tag is variable */
 168
 169         /* Word 7 */
 170         bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
 171         bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
 172         bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
 173         bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
 174
 175         /* Word 8 - abort_tag is variable */
 176
 177         /* Word 9  - reqtag is variable */
 178
 179         /* Word 10 - dbde, wqes is variable */
 180         bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
 181         bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
 182         bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
 183         bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
 184         bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
 185         bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
 186
 187         /* Word 11 */
 188         bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND);
 189         bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 190         bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
 191
 192         /* Word 12, 13, 14, 15 - is zero */
 193 }
 194
 195 /**
 196  * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry.
 197  * @pwqeq: Pointer to command iocb.
 198  * @xritag: Tag that  uniqely identifies the local exchange resource.
 199  * @opt: Option bits -
 200  *              bit 0 = inhibit sending abts on the link
 201  *
 202  * This function is called with hbalock held.
 203  **/
 204 void
 205 lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt)
 206 {
 207         union lpfc_wqe128 *wqe = &pwqeq->wqe;
 208
 209         /* WQEs are reused.  Clear stale data and set key fields to
 210          * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
 211          */
 212         memset(wqe, 0, sizeof(*wqe));
 213
 214         if (opt & INHIBIT_ABORT)
 215                 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
 216         /* Abort specified xri tag, with the mask deliberately zeroed */
 217         bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
 218
 219         bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
 220
 221         /* Abort the IO associated with this outstanding exchange ID. */
 222         wqe->abort_cmd.wqe_com.abort_tag = xritag;
 223
 224         /* iotag for the wqe completion. */
 225         bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag);
 226
 227         bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
 228         bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
 229
 230         bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND);
 231         bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1);
 232         bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 233 }
 234
 235 /**
 236  * lpfc_nvme_create_queue -
 237  * @pnvme_lport: Transport localport that LS is to be issued from
 238  * @lpfc_pnvme: Pointer to the driver's nvme instance data
 239  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
 240  * @qsize: Size of the queue in bytes
 241  * @handle: An opaque driver handle used in follow-up calls.
 242  *
 243  * Driver registers this routine to preallocate and initialize any
 244  * internal data structures to bind the @qidx to its internal IO queues.
 245  * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
 246  *
 247  * Return value :
 248  *   0 - Success
 249  *   -EINVAL - Unsupported input value.
 250  *   -ENOMEM - Could not alloc necessary memory
 251  **/
 252 static int
 253 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
 254                        unsigned int qidx, u16 qsize,
 255                        void **handle)
 256 {
 257         struct lpfc_nvme_lport *lport;
 258         struct lpfc_vport *vport;
 259         struct lpfc_nvme_qhandle *qhandle;
 260         char *str;
 261
 262         if (!pnvme_lport->private)
 263                 return -ENOMEM;
 264
 265         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
 266         vport = lport->vport;
 267         qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
 268         if (qhandle == NULL)
 269                 return -ENOMEM;
 270
 271         qhandle->cpu_id = raw_smp_processor_id();
 272         qhandle->qidx = qidx;
 273         /*
 274          * NVME qidx == 0 is the admin queue, so both admin queue
 275          * and first IO queue will use MSI-X vector and associated
 276          * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
 277          */
 278         if (qidx) {
 279                 str = "IO ";  /* IO queue */
 280                 qhandle->index = ((qidx - 1) %
 281                         lpfc_nvme_template.max_hw_queues);
 282         } else {
 283                 str = "ADM";  /* Admin queue */
 284                 qhandle->index = qidx;
 285         }
 286
 287         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
 288                          "6073 Binding %s HdwQueue %d  (cpu %d) to "
 289                          "hdw_queue %d qhandle x%px\n", str,
 290                          qidx, qhandle->cpu_id, qhandle->index, qhandle);
 291         *handle = (void *)qhandle;
 292         return 0;
 293 }
 294
 295 /**
 296  * lpfc_nvme_delete_queue -
 297  * @pnvme_lport: Transport localport that LS is to be issued from
 298  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
 299  * @handle: An opaque driver handle from lpfc_nvme_create_queue
 300  *
 301  * Driver registers this routine to free
 302  * any internal data structures to bind the @qidx to its internal
 303  * IO queues.
 304  *
 305  * Return value :
 306  *   0 - Success
 307  *   TODO:  What are the failure codes.
 308  **/
 309 static void
 310 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
 311                        unsigned int qidx,
 312                        void *handle)
 313 {
 314         struct lpfc_nvme_lport *lport;
 315         struct lpfc_vport *vport;
 316
 317         if (!pnvme_lport->private)
 318                 return;
 319
 320         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
 321         vport = lport->vport;
 322
 323         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
 324                         "6001 ENTER.  lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
 325                         lport, qidx, handle);
 326         kfree(handle);
 327 }
 328
 329 static void
 330 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
 331 {
 332         struct lpfc_nvme_lport *lport = localport->private;
 333
 334         lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
 335                          "6173 localport x%px delete complete\n",
 336                          lport);
 337
 338         /* release any threads waiting for the unreg to complete */
 339         if (lport->vport->localport)
 340                 complete(lport->lport_unreg_cmp);
 341 }
 342
 343 /* lpfc_nvme_remoteport_delete
 344  *
 345  * @remoteport: Pointer to an nvme transport remoteport instance.
 346  *
 347  * This is a template downcall.  NVME transport calls this function
 348  * when it has completed the unregistration of a previously
 349  * registered remoteport.
 350  *
 351  * Return value :
 352  * None
 353  */
 354 static void
 355 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
 356 {
 357         struct lpfc_nvme_rport *rport = remoteport->private;
 358         struct lpfc_vport *vport;
 359         struct lpfc_nodelist *ndlp;
 360
 361         ndlp = rport->ndlp;
 362         if (!ndlp)
 363                 goto rport_err;
 364
 365         vport = ndlp->vport;
 366         if (!vport)
 367                 goto rport_err;
 368
 369         /* Remove this rport from the lport's list - memory is owned by the
 370          * transport. Remove the ndlp reference for the NVME transport before
 371          * calling state machine to remove the node.
 372          */
 373         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
 374                         "6146 remoteport delete of remoteport x%px\n",
 375                         remoteport);
 376         spin_lock_irq(&vport->phba->hbalock);
 377
 378         /* The register rebind might have occurred before the delete
 379          * downcall.  Guard against this race.
 380          */
 381         if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
 382                 ndlp->nrport = NULL;
 383                 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
 384                 spin_unlock_irq(&vport->phba->hbalock);
 385
 386                 /* Remove original register reference. The host transport
 387                  * won't reference this rport/remoteport any further.
 388                  */
 389                 lpfc_nlp_put(ndlp);
 390         } else {
 391                 spin_unlock_irq(&vport->phba->hbalock);
 392         }
 393
 394  rport_err:
 395         return;
 396 }
 397
 398 /**
 399  * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
 400  * @phba: pointer to lpfc hba data structure.
 401  * @axchg: pointer to exchange context for the NVME LS request
 402  *
 403  * This routine is used for processing an asychronously received NVME LS
 404  * request. Any remaining validation is done and the LS is then forwarded
 405  * to the nvme-fc transport via nvme_fc_rcv_ls_req().
 406  *
 407  * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
 408  * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
 409  * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
 410  *
 411  * Returns 0 if LS was handled and delivered to the transport
 412  * Returns 1 if LS failed to be handled and should be dropped
 413  */
 414 int
 415 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
 416                         struct lpfc_async_xchg_ctx *axchg)
 417 {
 418 #if (IS_ENABLED(CONFIG_NVME_FC))
 419         struct lpfc_vport *vport;
 420         struct lpfc_nvme_rport *lpfc_rport;
 421         struct nvme_fc_remote_port *remoteport;
 422         struct lpfc_nvme_lport *lport;
 423         uint32_t *payload = axchg->payload;
 424         int rc;
 425
 426         vport = axchg->ndlp->vport;
 427         lpfc_rport = axchg->ndlp->nrport;
 428         if (!lpfc_rport)
 429                 return -EINVAL;
 430
 431         remoteport = lpfc_rport->remoteport;
 432         if (!vport->localport)
 433                 return -EINVAL;
 434
 435         lport = vport->localport->private;
 436         if (!lport)
 437                 return -EINVAL;
 438
 439         rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
 440                                 axchg->size);
 441
 442         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 443                         "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
 444                         "%08x %08x %08x\n",
 445                         axchg->size, rc,
 446                         *payload, *(payload+1), *(payload+2),
 447                         *(payload+3), *(payload+4), *(payload+5));
 448
 449         if (!rc)
 450                 return 0;
 451 #endif
 452         return 1;
 453 }
 454
 455 /**
 456  * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
 457  *        LS request.
 458  * @phba: Pointer to HBA context object
 459  * @vport: The local port that issued the LS
 460  * @cmdwqe: Pointer to driver command WQE object.
 461  * @wcqe: Pointer to driver response CQE object.
 462  *
 463  * This function is the generic completion handler for NVME LS requests.
 464  * The function updates any states and statistics, calls the transport
 465  * ls_req done() routine, then tears down the command and buffers used
 466  * for the LS request.
 467  **/
 468 void
 469 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba,  struct lpfc_vport *vport,
 470                         struct lpfc_iocbq *cmdwqe,
 471                         struct lpfc_wcqe_complete *wcqe)
 472 {
 473         struct nvmefc_ls_req *pnvme_lsreq;
 474         struct lpfc_dmabuf *buf_ptr;
 475         struct lpfc_nodelist *ndlp;
 476         uint32_t status;
 477
 478         pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
 479         ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
 480         status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
 481
 482         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
 483                          "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x "
 484                          "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
 485                          "ndlp:x%px\n",
 486                          pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
 487                          cmdwqe->sli4_xritag, status,
 488                          (wcqe->parameter & 0xffff),
 489                          cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
 490
 491         lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
 492                          cmdwqe->sli4_xritag, status, wcqe->parameter);
 493
 494         if (cmdwqe->context3) {
 495                 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
 496                 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
 497                 kfree(buf_ptr);
 498                 cmdwqe->context3 = NULL;
 499         }
 500         if (pnvme_lsreq->done)
 501                 pnvme_lsreq->done(pnvme_lsreq, status);
 502         else
 503                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 504                                  "6046 NVMEx cmpl without done call back? "
 505                                  "Data %px DID %x Xri: %x status %x\n",
 506                                 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
 507                                 cmdwqe->sli4_xritag, status);
 508         if (ndlp) {
 509                 lpfc_nlp_put(ndlp);
 510                 cmdwqe->context1 = NULL;
 511         }
 512         lpfc_sli_release_iocbq(phba, cmdwqe);
 513 }
 514
 515 static void
 516 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 517                        struct lpfc_wcqe_complete *wcqe)
 518 {
 519         struct lpfc_vport *vport = cmdwqe->vport;
 520         struct lpfc_nvme_lport *lport;
 521         uint32_t status;
 522
 523         status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
 524
 525         if (vport->localport) {
 526                 lport = (struct lpfc_nvme_lport *)vport->localport->private;
 527                 if (lport) {
 528                         atomic_inc(&lport->fc4NvmeLsCmpls);
 529                         if (status) {
 530                                 if (bf_get(lpfc_wcqe_c_xb, wcqe))
 531                                         atomic_inc(&lport->cmpl_ls_xb);
 532                                 atomic_inc(&lport->cmpl_ls_err);
 533                         }
 534                 }
 535         }
 536
 537         __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
 538 }
 539
 540 static int
 541 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
 542                   struct lpfc_dmabuf *inp,
 543                   struct nvmefc_ls_req *pnvme_lsreq,
 544                   void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
 545                                struct lpfc_wcqe_complete *),
 546                   struct lpfc_nodelist *ndlp, uint32_t num_entry,
 547                   uint32_t tmo, uint8_t retry)
 548 {
 549         struct lpfc_hba *phba = vport->phba;
 550         union lpfc_wqe128 *wqe;
 551         struct lpfc_iocbq *genwqe;
 552         struct ulp_bde64 *bpl;
 553         struct ulp_bde64 bde;
 554         int i, rc, xmit_len, first_len;
 555
 556         /* Allocate buffer for  command WQE */
 557         genwqe = lpfc_sli_get_iocbq(phba);
 558         if (genwqe == NULL)
 559                 return 1;
 560
 561         wqe = &genwqe->wqe;
 562         /* Initialize only 64 bytes */
 563         memset(wqe, 0, sizeof(union lpfc_wqe));
 564
 565         genwqe->context3 = (uint8_t *)bmp;
 566         genwqe->iocb_flag |= LPFC_IO_NVME_LS;
 567
 568         /* Save for completion so we can release these resources */
 569         genwqe->context1 = lpfc_nlp_get(ndlp);
 570         genwqe->context2 = (uint8_t *)pnvme_lsreq;
 571         /* Fill in payload, bp points to frame payload */
 572
 573         if (!tmo)
 574                 /* FC spec states we need 3 * ratov for CT requests */
 575                 tmo = (3 * phba->fc_ratov);
 576
 577         /* For this command calculate the xmit length of the request bde. */
 578         xmit_len = 0;
 579         first_len = 0;
 580         bpl = (struct ulp_bde64 *)bmp->virt;
 581         for (i = 0; i < num_entry; i++) {
 582                 bde.tus.w = bpl[i].tus.w;
 583                 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
 584                         break;
 585                 xmit_len += bde.tus.f.bdeSize;
 586                 if (i == 0)
 587                         first_len = xmit_len;
 588         }
 589
 590         genwqe->rsvd2 = num_entry;
 591         genwqe->hba_wqidx = 0;
 592
 593         /* Words 0 - 2 */
 594         wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 595         wqe->generic.bde.tus.f.bdeSize = first_len;
 596         wqe->generic.bde.addrLow = bpl[0].addrLow;
 597         wqe->generic.bde.addrHigh = bpl[0].addrHigh;
 598
 599         /* Word 3 */
 600         wqe->gen_req.request_payload_len = first_len;
 601
 602         /* Word 4 */
 603
 604         /* Word 5 */
 605         bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
 606         bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
 607         bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
 608         bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
 609         bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
 610
 611         /* Word 6 */
 612         bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
 613                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
 614         bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
 615
 616         /* Word 7 */
 617         bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1));
 618         bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
 619         bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
 620         bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
 621
 622         /* Word 8 */
 623         wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
 624
 625         /* Word 9 */
 626         bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
 627
 628         /* Word 10 */
 629         bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
 630         bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
 631         bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
 632         bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
 633         bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
 634
 635         /* Word 11 */
 636         bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
 637         bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
 638
 639
 640         /* Issue GEN REQ WQE for NPORT <did> */
 641         genwqe->wqe_cmpl = cmpl;
 642         genwqe->iocb_cmpl = NULL;
 643         genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
 644         genwqe->vport = vport;
 645         genwqe->retry = retry;
 646
 647         lpfc_nvmeio_data(phba, "NVME LS  XMIT: xri x%x iotag x%x to x%06x\n",
 648                          genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
 649
 650         rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
 651         if (rc) {
 652                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 653                                  "6045 Issue GEN REQ WQE to NPORT x%x "
 654                                  "Data: x%x x%x  rc x%x\n",
 655                                  ndlp->nlp_DID, genwqe->iotag,
 656                                  vport->port_state, rc);
 657                 lpfc_sli_release_iocbq(phba, genwqe);
 658                 return 1;
 659         }
 660
 661         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
 662                          "6050 Issue GEN REQ WQE to NPORT x%x "
 663                          "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
 664                          "bmp:x%px xmit:%d 1st:%d\n",
 665                          ndlp->nlp_DID, genwqe->sli4_xritag,
 666                          vport->port_state,
 667                          genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
 668         return 0;
 669 }
 670
 671
 672 /**
 673  * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
 674  * @vport: The local port issuing the LS
 675  * @ndlp: The remote port to send the LS to
 676  * @pnvme_lsreq: Pointer to LS request structure from the transport
 677  * @gen_req_cmp: Completion call-back
 678  *
 679  * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
 680  * WQE to perform the LS operation.
 681  *
 682  * Return value :
 683  *   0 - Success
 684  *   non-zero: various error codes, in form of -Exxx
 685  **/
 686 int
 687 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 688                       struct nvmefc_ls_req *pnvme_lsreq,
 689                       void (*gen_req_cmp)(struct lpfc_hba *phba,
 690                                 struct lpfc_iocbq *cmdwqe,
 691                                 struct lpfc_wcqe_complete *wcqe))
 692 {
 693         struct lpfc_dmabuf *bmp;
 694         struct ulp_bde64 *bpl;
 695         int ret;
 696         uint16_t ntype, nstate;
 697
 698         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
 699                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 700                                  "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
 701                                  "LS Req\n",
 702                                  ndlp);
 703                 return -ENODEV;
 704         }
 705
 706         ntype = ndlp->nlp_type;
 707         nstate = ndlp->nlp_state;
 708         if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
 709             (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
 710                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 711                                  "6088 NVMEx LS REQ: Fail DID x%06x not "
 712                                  "ready for IO. Type x%x, State x%x\n",
 713                                  ndlp->nlp_DID, ntype, nstate);
 714                 return -ENODEV;
 715         }
 716
 717         /*
 718          * there are two dma buf in the request, actually there is one and
 719          * the second one is just the start address + cmd size.
 720          * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
 721          * in a lpfc_dmabuf struct. When freeing we just free the wrapper
 722          * because the nvem layer owns the data bufs.
 723          * We do not have to break these packets open, we don't care what is
 724          * in them. And we do not have to look at the resonse data, we only
 725          * care that we got a response. All of the caring is going to happen
 726          * in the nvme-fc layer.
 727          */
 728
 729         bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
 730         if (!bmp) {
 731                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 732                                  "6044 NVMEx LS REQ: Could not alloc LS buf "
 733                                  "for DID %x\n",
 734                                  ndlp->nlp_DID);
 735                 return -ENOMEM;
 736         }
 737
 738         bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
 739         if (!bmp->virt) {
 740                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 741                                  "6042 NVMEx LS REQ: Could not alloc mbuf "
 742                                  "for DID %x\n",
 743                                  ndlp->nlp_DID);
 744                 kfree(bmp);
 745                 return -ENOMEM;
 746         }
 747
 748         INIT_LIST_HEAD(&bmp->list);
 749
 750         bpl = (struct ulp_bde64 *)bmp->virt;
 751         bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
 752         bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
 753         bpl->tus.f.bdeFlags = 0;
 754         bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
 755         bpl->tus.w = le32_to_cpu(bpl->tus.w);
 756         bpl++;
 757
 758         bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
 759         bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
 760         bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
 761         bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
 762         bpl->tus.w = le32_to_cpu(bpl->tus.w);
 763
 764         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
 765                         "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
 766                         "rqstlen:%d rsplen:%d %pad %pad\n",
 767                         ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
 768                         pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
 769                         &pnvme_lsreq->rspdma);
 770
 771         ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
 772                                 pnvme_lsreq, gen_req_cmp, ndlp, 2,
 773                                 LPFC_NVME_LS_TIMEOUT, 0);
 774         if (ret != WQE_SUCCESS) {
 775                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 776                                  "6052 NVMEx REQ: EXIT. issue ls wqe failed "
 777                                  "lsreq x%px Status %x DID %x\n",
 778                                  pnvme_lsreq, ret, ndlp->nlp_DID);
 779                 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
 780                 kfree(bmp);
 781                 return -EIO;
 782         }
 783
 784         return 0;
 785 }
 786
 787 /**
 788  * lpfc_nvme_ls_req - Issue an NVME Link Service request
 789  * @pnvme_lport: Transport localport that LS is to be issued from.
 790  * @nvme_rport: Transport remoteport that LS is to be sent to.
 791  * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
 792  *
 793  * Driver registers this routine to handle any link service request
 794  * from the nvme_fc transport to a remote nvme-aware port.
 795  *
 796  * Return value :
 797  *   0 - Success
 798  *   non-zero: various error codes, in form of -Exxx
 799  **/
 800 static int
 801 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
 802                  struct nvme_fc_remote_port *pnvme_rport,
 803                  struct nvmefc_ls_req *pnvme_lsreq)
 804 {
 805         struct lpfc_nvme_lport *lport;
 806         struct lpfc_nvme_rport *rport;
 807         struct lpfc_vport *vport;
 808         int ret;
 809
 810         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
 811         rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
 812         if (unlikely(!lport) || unlikely(!rport))
 813                 return -EINVAL;
 814
 815         vport = lport->vport;
 816         if (vport->load_flag & FC_UNLOADING)
 817                 return -ENODEV;
 818
 819         atomic_inc(&lport->fc4NvmeLsRequests);
 820
 821         ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
 822                                  lpfc_nvme_ls_req_cmp);
 823         if (ret)
 824                 atomic_inc(&lport->xmt_ls_err);
 825
 826         return ret;
 827 }
 828
 829 /**
 830  * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
 831  *         NVME LS request
 832  * @vport: The local port that issued the LS
 833  * @ndlp: The remote port the LS was sent to
 834  * @pnvme_lsreq: Pointer to LS request structure from the transport
 835  *
 836  * The driver validates the ndlp, looks for the LS, and aborts the
 837  * LS if found.
 838  *
 839  * Returns:
 840  * 0 : if LS found and aborted
 841  * non-zero: various error conditions in form -Exxx
 842  **/
 843 int
 844 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 845                         struct nvmefc_ls_req *pnvme_lsreq)
 846 {
 847         struct lpfc_hba *phba = vport->phba;
 848         struct lpfc_sli_ring *pring;
 849         struct lpfc_iocbq *wqe, *next_wqe;
 850         bool foundit = false;
 851
 852         if (!ndlp) {
 853                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 854                                 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
 855                                 "x%06x, Failing LS Req\n",
 856                                 ndlp, ndlp ? ndlp->nlp_DID : 0);
 857                 return -EINVAL;
 858         }
 859
 860         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
 861                          "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
 862                          "x%p rqstlen:%d rsplen:%d %pad %pad\n",
 863                          pnvme_lsreq, pnvme_lsreq->rqstlen,
 864                          pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
 865                          &pnvme_lsreq->rspdma);
 866
 867         /*
 868          * Lock the ELS ring txcmplq and look for the wqe that matches
 869          * this ELS. If found, issue an abort on the wqe.
 870          */
 871         pring = phba->sli4_hba.nvmels_wq->pring;
 872         spin_lock_irq(&phba->hbalock);
 873         spin_lock(&pring->ring_lock);
 874         list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
 875                 if (wqe->context2 == pnvme_lsreq) {
 876                         wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
 877                         foundit = true;
 878                         break;
 879                 }
 880         }
 881         spin_unlock(&pring->ring_lock);
 882
 883         if (foundit)
 884                 lpfc_sli_issue_abort_iotag(phba, pring, wqe);
 885         spin_unlock_irq(&phba->hbalock);
 886
 887         if (foundit)
 888                 return 0;
 889
 890         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
 891                          "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n",
 892                          pnvme_lsreq);
 893         return -EINVAL;
 894 }
 895
 896 static int
 897 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
 898                      struct nvme_fc_remote_port *remoteport,
 899                      struct nvmefc_ls_rsp *ls_rsp)
 900 {
 901         struct lpfc_async_xchg_ctx *axchg =
 902                 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
 903         struct lpfc_nvme_lport *lport;
 904         int rc;
 905
 906         if (axchg->phba->pport->load_flag & FC_UNLOADING)
 907                 return -ENODEV;
 908
 909         lport = (struct lpfc_nvme_lport *)localport->private;
 910
 911         rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
 912
 913         if (rc) {
 914                 /*
 915                  * unless the failure is due to having already sent
 916                  * the response, an abort will be generated for the
 917                  * exchange if the rsp can't be sent.
 918                  */
 919                 if (rc != -EALREADY)
 920                         atomic_inc(&lport->xmt_ls_abort);
 921                 return rc;
 922         }
 923
 924         return 0;
 925 }
 926
 927 /**
 928  * lpfc_nvme_ls_abort - Abort a prior NVME LS request
 929  * @pnvme_lport: Transport localport that LS is to be issued from.
 930  * @pnvme_rport: Transport remoteport that LS is to be sent to.
 931  * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
 932  *
 933  * Driver registers this routine to abort a NVME LS request that is
 934  * in progress (from the transports perspective).
 935  **/
 936 static void
 937 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
 938                    struct nvme_fc_remote_port *pnvme_rport,
 939                    struct nvmefc_ls_req *pnvme_lsreq)
 940 {
 941         struct lpfc_nvme_lport *lport;
 942         struct lpfc_vport *vport;
 943         struct lpfc_hba *phba;
 944         struct lpfc_nodelist *ndlp;
 945         int ret;
 946
 947         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
 948         if (unlikely(!lport))
 949                 return;
 950         vport = lport->vport;
 951         phba = vport->phba;
 952
 953         if (vport->load_flag & FC_UNLOADING)
 954                 return;
 955
 956         ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
 957
 958         ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
 959         if (!ret)
 960                 atomic_inc(&lport->xmt_ls_abort);
 961 }
 962
 963 /* Fix up the existing sgls for NVME IO. */
 964 static inline void
 965 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
 966                        struct lpfc_io_buf *lpfc_ncmd,
 967                        struct nvmefc_fcp_req *nCmd)
 968 {
 969         struct lpfc_hba  *phba = vport->phba;
 970         struct sli4_sge *sgl;
 971         union lpfc_wqe128 *wqe;
 972         uint32_t *wptr, *dptr;
 973
 974         /*
 975          * Get a local pointer to the built-in wqe and correct
 976          * the cmd size to match NVME's 96 bytes and fix
 977          * the dma address.
 978          */
 979
 980         wqe = &lpfc_ncmd->cur_iocbq.wqe;
 981
 982         /*
 983          * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
 984          * match NVME.  NVME sends 96 bytes. Also, use the
 985          * nvme commands command and response dma addresses
 986          * rather than the virtual memory to ease the restore
 987          * operation.
 988          */
 989         sgl = lpfc_ncmd->dma_sgl;
 990         sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
 991         if (phba->cfg_nvme_embed_cmd) {
 992                 sgl->addr_hi = 0;
 993                 sgl->addr_lo = 0;
 994
 995                 /* Word 0-2 - NVME CMND IU (embedded payload) */
 996                 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
 997                 wqe->generic.bde.tus.f.bdeSize = 56;
 998                 wqe->generic.bde.addrHigh = 0;
 999                 wqe->generic.bde.addrLow =  64;  /* Word 16 */
1000
1001                 /* Word 10  - dbde is 0, wqes is 1 in template */
1002
1003                 /*
1004                  * Embed the payload in the last half of the WQE
1005                  * WQE words 16-30 get the NVME CMD IU payload
1006                  *
1007                  * WQE words 16-19 get payload Words 1-4
1008                  * WQE words 20-21 get payload Words 6-7
1009                  * WQE words 22-29 get payload Words 16-23
1010                  */
1011                 wptr = &wqe->words[16];  /* WQE ptr */
1012                 dptr = (uint32_t *)nCmd->cmdaddr;  /* payload ptr */
1013                 dptr++;                 /* Skip Word 0 in payload */
1014
1015                 *wptr++ = *dptr++;      /* Word 1 */
1016                 *wptr++ = *dptr++;      /* Word 2 */
1017                 *wptr++ = *dptr++;      /* Word 3 */
1018                 *wptr++ = *dptr++;      /* Word 4 */
1019                 dptr++;                 /* Skip Word 5 in payload */
1020                 *wptr++ = *dptr++;      /* Word 6 */
1021                 *wptr++ = *dptr++;      /* Word 7 */
1022                 dptr += 8;              /* Skip Words 8-15 in payload */
1023                 *wptr++ = *dptr++;      /* Word 16 */
1024                 *wptr++ = *dptr++;      /* Word 17 */
1025                 *wptr++ = *dptr++;      /* Word 18 */
1026                 *wptr++ = *dptr++;      /* Word 19 */
1027                 *wptr++ = *dptr++;      /* Word 20 */
1028                 *wptr++ = *dptr++;      /* Word 21 */
1029                 *wptr++ = *dptr++;      /* Word 22 */
1030                 *wptr   = *dptr;        /* Word 23 */
1031         } else {
1032                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
1033                 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
1034
1035                 /* Word 0-2 - NVME CMND IU Inline BDE */
1036                 wqe->generic.bde.tus.f.bdeFlags =  BUFF_TYPE_BDE_64;
1037                 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
1038                 wqe->generic.bde.addrHigh = sgl->addr_hi;
1039                 wqe->generic.bde.addrLow =  sgl->addr_lo;
1040
1041                 /* Word 10 */
1042                 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
1043                 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
1044         }
1045
1046         sgl++;
1047
1048         /* Setup the physical region for the FCP RSP */
1049         sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
1050         sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
1051         sgl->word2 = le32_to_cpu(sgl->word2);
1052         if (nCmd->sg_cnt)
1053                 bf_set(lpfc_sli4_sge_last, sgl, 0);
1054         else
1055                 bf_set(lpfc_sli4_sge_last, sgl, 1);
1056         sgl->word2 = cpu_to_le32(sgl->word2);
1057         sgl->sge_len = cpu_to_le32(nCmd->rsplen);
1058 }
1059
1060
1061 /*
1062  * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
1063  *
1064  * Driver registers this routine as it io request handler.  This
1065  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1066  * data structure to the rport indicated in @lpfc_nvme_rport.
1067  *
1068  * Return value :
1069  *   0 - Success
1070  *   TODO: What are the failure codes.
1071  **/
1072 static void
1073 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
1074                           struct lpfc_wcqe_complete *wcqe)
1075 {
1076         struct lpfc_io_buf *lpfc_ncmd =
1077                 (struct lpfc_io_buf *)pwqeIn->context1;
1078         struct lpfc_vport *vport = pwqeIn->vport;
1079         struct nvmefc_fcp_req *nCmd;
1080         struct nvme_fc_ersp_iu *ep;
1081         struct nvme_fc_cmd_iu *cp;
1082         struct lpfc_nodelist *ndlp;
1083         struct lpfc_nvme_fcpreq_priv *freqpriv;
1084         struct lpfc_nvme_lport *lport;
1085         uint32_t code, status, idx;
1086         uint16_t cid, sqhd, data;
1087         uint32_t *ptr;
1088 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1089         int cpu;
1090 #endif
1091
1092         /* Sanity check on return of outstanding command */
1093         if (!lpfc_ncmd) {
1094                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1095                                  "6071 Null lpfc_ncmd pointer. No "
1096                                  "release, skip completion\n");
1097                 return;
1098         }
1099
1100         /* Guard against abort handler being called at same time */
1101         spin_lock(&lpfc_ncmd->buf_lock);
1102
1103         if (!lpfc_ncmd->nvmeCmd) {
1104                 spin_unlock(&lpfc_ncmd->buf_lock);
1105                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1106                                  "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
1107                                  "nvmeCmd x%px\n",
1108                                  lpfc_ncmd, lpfc_ncmd->nvmeCmd);
1109
1110                 /* Release the lpfc_ncmd regardless of the missing elements. */
1111                 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1112                 return;
1113         }
1114         nCmd = lpfc_ncmd->nvmeCmd;
1115         status = bf_get(lpfc_wcqe_c_status, wcqe);
1116
1117         idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
1118         phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
1119
1120         if (unlikely(status && vport->localport)) {
1121                 lport = (struct lpfc_nvme_lport *)vport->localport->private;
1122                 if (lport) {
1123                         if (bf_get(lpfc_wcqe_c_xb, wcqe))
1124                                 atomic_inc(&lport->cmpl_fcp_xb);
1125                         atomic_inc(&lport->cmpl_fcp_err);
1126                 }
1127         }
1128
1129         lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
1130                          lpfc_ncmd->cur_iocbq.sli4_xritag,
1131                          status, wcqe->parameter);
1132         /*
1133          * Catch race where our node has transitioned, but the
1134          * transport is still transitioning.
1135          */
1136         ndlp = lpfc_ncmd->ndlp;
1137         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1138                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1139                                  "6062 Ignoring NVME cmpl.  No ndlp\n");
1140                 goto out_err;
1141         }
1142
1143         code = bf_get(lpfc_wcqe_c_code, wcqe);
1144         if (code == CQE_CODE_NVME_ERSP) {
1145                 /* For this type of CQE, we need to rebuild the rsp */
1146                 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1147
1148                 /*
1149                  * Get Command Id from cmd to plug into response. This
1150                  * code is not needed in the next NVME Transport drop.
1151                  */
1152                 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1153                 cid = cp->sqe.common.command_id;
1154
1155                 /*
1156                  * RSN is in CQE word 2
1157                  * SQHD is in CQE Word 3 bits 15:0
1158                  * Cmd Specific info is in CQE Word 1
1159                  * and in CQE Word 0 bits 15:0
1160                  */
1161                 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1162
1163                 /* Now lets build the NVME ERSP IU */
1164                 ep->iu_len = cpu_to_be16(8);
1165                 ep->rsn = wcqe->parameter;
1166                 ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1167                 ep->rsvd12 = 0;
1168                 ptr = (uint32_t *)&ep->cqe.result.u64;
1169                 *ptr++ = wcqe->total_data_placed;
1170                 data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1171                 *ptr = (uint32_t)data;
1172                 ep->cqe.sq_head = sqhd;
1173                 ep->cqe.sq_id =  nCmd->sqid;
1174                 ep->cqe.command_id = cid;
1175                 ep->cqe.status = 0;
1176
1177                 lpfc_ncmd->status = IOSTAT_SUCCESS;
1178                 lpfc_ncmd->result = 0;
1179                 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1180                 nCmd->transferred_length = nCmd->payload_length;
1181         } else {
1182                 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);
1183                 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1184
1185                 /* For NVME, the only failure path that results in an
1186                  * IO error is when the adapter rejects it.  All other
1187                  * conditions are a success case and resolved by the
1188                  * transport.
1189                  * IOSTAT_FCP_RSP_ERROR means:
1190                  * 1. Length of data received doesn't match total
1191                  *    transfer length in WQE
1192                  * 2. If the RSP payload does NOT match these cases:
1193                  *    a. RSP length 12/24 bytes and all zeros
1194                  *    b. NVME ERSP
1195                  */
1196                 switch (lpfc_ncmd->status) {
1197                 case IOSTAT_SUCCESS:
1198                         nCmd->transferred_length = wcqe->total_data_placed;
1199                         nCmd->rcv_rsplen = 0;
1200                         nCmd->status = 0;
1201                         break;
1202                 case IOSTAT_FCP_RSP_ERROR:
1203                         nCmd->transferred_length = wcqe->total_data_placed;
1204                         nCmd->rcv_rsplen = wcqe->parameter;
1205                         nCmd->status = 0;
1206                         /* Sanity check */
1207                         if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
1208                                 break;
1209                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1210                                          "6081 NVME Completion Protocol Error: "
1211                                          "xri %x status x%x result x%x "
1212                                          "placed x%x\n",
1213                                          lpfc_ncmd->cur_iocbq.sli4_xritag,
1214                                          lpfc_ncmd->status, lpfc_ncmd->result,
1215                                          wcqe->total_data_placed);
1216                         break;
1217                 case IOSTAT_LOCAL_REJECT:
1218                         /* Let fall through to set command final state. */
1219                         if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1220                                 lpfc_printf_vlog(vport, KERN_INFO,
1221                                          LOG_NVME_IOERR,
1222                                          "6032 Delay Aborted cmd x%px "
1223                                          "nvme cmd x%px, xri x%x, "
1224                                          "xb %d\n",
1225                                          lpfc_ncmd, nCmd,
1226                                          lpfc_ncmd->cur_iocbq.sli4_xritag,
1227                                          bf_get(lpfc_wcqe_c_xb, wcqe));
1228                         fallthrough;
1229                 default:
1230 out_err:
1231                         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1232                                          "6072 NVME Completion Error: xri %x "
1233                                          "status x%x result x%x [x%x] "
1234                                          "placed x%x\n",
1235                                          lpfc_ncmd->cur_iocbq.sli4_xritag,
1236                                          lpfc_ncmd->status, lpfc_ncmd->result,
1237                                          wcqe->parameter,
1238                                          wcqe->total_data_placed);
1239                         nCmd->transferred_length = 0;
1240                         nCmd->rcv_rsplen = 0;
1241                         nCmd->status = NVME_SC_INTERNAL;
1242                 }
1243         }
1244
1245         /* pick up SLI4 exhange busy condition */
1246         if (bf_get(lpfc_wcqe_c_xb, wcqe))
1247                 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1248         else
1249                 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1250
1251         /* Update stats and complete the IO.  There is
1252          * no need for dma unprep because the nvme_transport
1253          * owns the dma address.
1254          */
1255 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1256         if (lpfc_ncmd->ts_cmd_start) {
1257                 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1258                 lpfc_ncmd->ts_data_io = ktime_get_ns();
1259                 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1260                 lpfc_io_ktime(phba, lpfc_ncmd);
1261         }
1262         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1263                 cpu = raw_smp_processor_id();
1264                 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1265                 if (lpfc_ncmd->cpu != cpu)
1266                         lpfc_printf_vlog(vport,
1267                                          KERN_INFO, LOG_NVME_IOERR,
1268                                          "6701 CPU Check cmpl: "
1269                                          "cpu %d expect %d\n",
1270                                          cpu, lpfc_ncmd->cpu);
1271         }
1272 #endif
1273
1274         /* NVME targets need completion held off until the abort exchange
1275          * completes unless the NVME Rport is getting unregistered.
1276          */
1277
1278         if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1279                 freqpriv = nCmd->private;
1280                 freqpriv->nvme_buf = NULL;
1281                 lpfc_ncmd->nvmeCmd = NULL;
1282                 spin_unlock(&lpfc_ncmd->buf_lock);
1283                 nCmd->done(nCmd);
1284         } else
1285                 spin_unlock(&lpfc_ncmd->buf_lock);
1286
1287         /* Call release with XB=1 to queue the IO into the abort list. */
1288         lpfc_release_nvme_buf(phba, lpfc_ncmd);
1289 }
1290
1291
1292 /**
1293  * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1294  * @vport: pointer to a host virtual N_Port data structure
1295  * @lpfcn_cmd: Pointer to lpfc scsi command
1296  * @pnode: pointer to a node-list data structure
1297  * @cstat: pointer to the control status structure
1298  *
1299  * Driver registers this routine as it io request handler.  This
1300  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1301  * data structure to the rport indicated in @lpfc_nvme_rport.
1302  *
1303  * Return value :
1304  *   0 - Success
1305  *   TODO: What are the failure codes.
1306  **/
1307 static int
1308 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1309                       struct lpfc_io_buf *lpfc_ncmd,
1310                       struct lpfc_nodelist *pnode,
1311                       struct lpfc_fc4_ctrl_stat *cstat)
1312 {
1313         struct lpfc_hba *phba = vport->phba;
1314         struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1315         struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
1316         union lpfc_wqe128 *wqe = &pwqeq->wqe;
1317         uint32_t req_len;
1318
1319         if (!NLP_CHK_NODE_ACT(pnode))
1320                 return -EINVAL;
1321
1322         /*
1323          * There are three possibilities here - use scatter-gather segment, use
1324          * the single mapping, or neither.
1325          */
1326         if (nCmd->sg_cnt) {
1327                 if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1328                         /* From the iwrite template, initialize words 7 - 11 */
1329                         memcpy(&wqe->words[7],
1330                                &lpfc_iwrite_cmd_template.words[7],
1331                                sizeof(uint32_t) * 5);
1332
1333                         /* Word 4 */
1334                         wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1335
1336                         /* Word 5 */
1337                         if ((phba->cfg_nvme_enable_fb) &&
1338                             (pnode->nlp_flag & NLP_FIRSTBURST)) {
1339                                 req_len = lpfc_ncmd->nvmeCmd->payload_length;
1340                                 if (req_len < pnode->nvme_fb_size)
1341                                         wqe->fcp_iwrite.initial_xfer_len =
1342                                                 req_len;
1343                                 else
1344                                         wqe->fcp_iwrite.initial_xfer_len =
1345                                                 pnode->nvme_fb_size;
1346                         } else {
1347                                 wqe->fcp_iwrite.initial_xfer_len = 0;
1348                         }
1349                         cstat->output_requests++;
1350                 } else {
1351                         /* From the iread template, initialize words 7 - 11 */
1352                         memcpy(&wqe->words[7],
1353                                &lpfc_iread_cmd_template.words[7],
1354                                sizeof(uint32_t) * 5);
1355
1356                         /* Word 4 */
1357                         wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1358
1359                         /* Word 5 */
1360                         wqe->fcp_iread.rsrvd5 = 0;
1361
1362                         cstat->input_requests++;
1363                 }
1364         } else {
1365                 /* From the icmnd template, initialize words 4 - 11 */
1366                 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1367                        sizeof(uint32_t) * 8);
1368                 cstat->control_requests++;
1369         }
1370
1371         if (pnode->nlp_nvme_info & NLP_NVME_NSLER)
1372                 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1373         /*
1374          * Finish initializing those WQE fields that are independent
1375          * of the nvme_cmnd request_buffer
1376          */
1377
1378         /* Word 3 */
1379         bf_set(payload_offset_len, &wqe->fcp_icmd,
1380                (nCmd->rsplen + nCmd->cmdlen));
1381
1382         /* Word 6 */
1383         bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1384                phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1385         bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1386
1387         /* Word 8 */
1388         wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1389
1390         /* Word 9 */
1391         bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1392
1393         /* Words 13 14 15 are for PBDE support */
1394
1395         pwqeq->vport = vport;
1396         return 0;
1397 }
1398
1399
1400 /**
1401  * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1402  * @vport: pointer to a host virtual N_Port data structure
1403  * @lpfcn_cmd: Pointer to lpfc scsi command
1404  *
1405  * Driver registers this routine as it io request handler.  This
1406  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1407  * data structure to the rport indicated in @lpfc_nvme_rport.
1408  *
1409  * Return value :
1410  *   0 - Success
1411  *   TODO: What are the failure codes.
1412  **/
1413 static int
1414 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1415                       struct lpfc_io_buf *lpfc_ncmd)
1416 {
1417         struct lpfc_hba *phba = vport->phba;
1418         struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1419         union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1420         struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1421         struct sli4_hybrid_sgl *sgl_xtra = NULL;
1422         struct scatterlist *data_sg;
1423         struct sli4_sge *first_data_sgl;
1424         struct ulp_bde64 *bde;
1425         dma_addr_t physaddr = 0;
1426         uint32_t num_bde = 0;
1427         uint32_t dma_len = 0;
1428         uint32_t dma_offset = 0;
1429         int nseg, i, j;
1430         bool lsp_just_set = false;
1431
1432         /* Fix up the command and response DMA stuff. */
1433         lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1434
1435         /*
1436          * There are three possibilities here - use scatter-gather segment, use
1437          * the single mapping, or neither.
1438          */
1439         if (nCmd->sg_cnt) {
1440                 /*
1441                  * Jump over the cmd and rsp SGEs.  The fix routine
1442                  * has already adjusted for this.
1443                  */
1444                 sgl += 2;
1445
1446                 first_data_sgl = sgl;
1447                 lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1448                 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1449                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1450                                         "6058 Too many sg segments from "
1451                                         "NVME Transport.  Max %d, "
1452                                         "nvmeIO sg_cnt %d\n",
1453                                         phba->cfg_nvme_seg_cnt + 1,
1454                                         lpfc_ncmd->seg_cnt);
1455                         lpfc_ncmd->seg_cnt = 0;
1456                         return 1;
1457                 }
1458
1459                 /*
1460                  * The driver established a maximum scatter-gather segment count
1461                  * during probe that limits the number of sg elements in any
1462                  * single nvme command.  Just run through the seg_cnt and format
1463                  * the sge's.
1464                  */
1465                 nseg = nCmd->sg_cnt;
1466                 data_sg = nCmd->first_sgl;
1467
1468                 /* for tracking the segment boundaries */
1469                 j = 2;
1470                 for (i = 0; i < nseg; i++) {
1471                         if (data_sg == NULL) {
1472                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1473                                                 "6059 dptr err %d, nseg %d\n",
1474                                                 i, nseg);
1475                                 lpfc_ncmd->seg_cnt = 0;
1476                                 return 1;
1477                         }
1478
1479                         sgl->word2 = 0;
1480                         if ((num_bde + 1) == nseg) {
1481                                 bf_set(lpfc_sli4_sge_last, sgl, 1);
1482                                 bf_set(lpfc_sli4_sge_type, sgl,
1483                                        LPFC_SGE_TYPE_DATA);
1484                         } else {
1485                                 bf_set(lpfc_sli4_sge_last, sgl, 0);
1486
1487                                 /* expand the segment */
1488                                 if (!lsp_just_set &&
1489                                     !((j + 1) % phba->border_sge_num) &&
1490                                     ((nseg - 1) != i)) {
1491                                         /* set LSP type */
1492                                         bf_set(lpfc_sli4_sge_type, sgl,
1493                                                LPFC_SGE_TYPE_LSP);
1494
1495                                         sgl_xtra = lpfc_get_sgl_per_hdwq(
1496                                                         phba, lpfc_ncmd);
1497
1498                                         if (unlikely(!sgl_xtra)) {
1499                                                 lpfc_ncmd->seg_cnt = 0;
1500                                                 return 1;
1501                                         }
1502                                         sgl->addr_lo = cpu_to_le32(putPaddrLow(
1503                                                        sgl_xtra->dma_phys_sgl));
1504                                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1505                                                        sgl_xtra->dma_phys_sgl));
1506
1507                                 } else {
1508                                         bf_set(lpfc_sli4_sge_type, sgl,
1509                                                LPFC_SGE_TYPE_DATA);
1510                                 }
1511                         }
1512
1513                         if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1514                                      LPFC_SGE_TYPE_LSP)) {
1515                                 if ((nseg - 1) == i)
1516                                         bf_set(lpfc_sli4_sge_last, sgl, 1);
1517
1518                                 physaddr = data_sg->dma_address;
1519                                 dma_len = data_sg->length;
1520                                 sgl->addr_lo = cpu_to_le32(
1521                                                          putPaddrLow(physaddr));
1522                                 sgl->addr_hi = cpu_to_le32(
1523                                                         putPaddrHigh(physaddr));
1524
1525                                 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1526                                 sgl->word2 = cpu_to_le32(sgl->word2);
1527                                 sgl->sge_len = cpu_to_le32(dma_len);
1528
1529                                 dma_offset += dma_len;
1530                                 data_sg = sg_next(data_sg);
1531
1532                                 sgl++;
1533
1534                                 lsp_just_set = false;
1535                         } else {
1536                                 sgl->word2 = cpu_to_le32(sgl->word2);
1537
1538                                 sgl->sge_len = cpu_to_le32(
1539                                                      phba->cfg_sg_dma_buf_size);
1540
1541                                 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1542                                 i = i - 1;
1543
1544                                 lsp_just_set = true;
1545                         }
1546
1547                         j++;
1548                 }
1549                 if (phba->cfg_enable_pbde) {
1550                         /* Use PBDE support for first SGL only, offset == 0 */
1551                         /* Words 13-15 */
1552                         bde = (struct ulp_bde64 *)
1553                                 &wqe->words[13];
1554                         bde->addrLow = first_data_sgl->addr_lo;
1555                         bde->addrHigh = first_data_sgl->addr_hi;
1556                         bde->tus.f.bdeSize =
1557                                 le32_to_cpu(first_data_sgl->sge_len);
1558                         bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1559                         bde->tus.w = cpu_to_le32(bde->tus.w);
1560                         /* wqe_pbde is 1 in template */
1561                 } else {
1562                         memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1563                         bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
1564                 }
1565
1566         } else {
1567                 lpfc_ncmd->seg_cnt = 0;
1568
1569                 /* For this clause to be valid, the payload_length
1570                  * and sg_cnt must zero.
1571                  */
1572                 if (nCmd->payload_length != 0) {
1573                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1574                                         "6063 NVME DMA Prep Err: sg_cnt %d "
1575                                         "payload_length x%x\n",
1576                                         nCmd->sg_cnt, nCmd->payload_length);
1577                         return 1;
1578                 }
1579         }
1580         return 0;
1581 }
1582
1583 /**
1584  * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1585  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1586  * @lpfc_nvme_lport: Pointer to the driver's local port data
1587  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1588  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1589  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1590  *
1591  * Driver registers this routine as it io request handler.  This
1592  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1593  * data structure to the rport
1594  indicated in @lpfc_nvme_rport.
1595  *
1596  * Return value :
1597  *   0 - Success
1598  *   TODO: What are the failure codes.
1599  **/
1600 static int
1601 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1602                         struct nvme_fc_remote_port *pnvme_rport,
1603                         void *hw_queue_handle,
1604                         struct nvmefc_fcp_req *pnvme_fcreq)
1605 {
1606         int ret = 0;
1607         int expedite = 0;
1608         int idx, cpu;
1609         struct lpfc_nvme_lport *lport;
1610         struct lpfc_fc4_ctrl_stat *cstat;
1611         struct lpfc_vport *vport;
1612         struct lpfc_hba *phba;
1613         struct lpfc_nodelist *ndlp;
1614         struct lpfc_io_buf *lpfc_ncmd;
1615         struct lpfc_nvme_rport *rport;
1616         struct lpfc_nvme_qhandle *lpfc_queue_info;
1617         struct lpfc_nvme_fcpreq_priv *freqpriv;
1618         struct nvme_common_command *sqe;
1619 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1620         uint64_t start = 0;
1621 #endif
1622
1623         /* Validate pointers. LLDD fault handling with transport does
1624          * have timing races.
1625          */
1626         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1627         if (unlikely(!lport)) {
1628                 ret = -EINVAL;
1629                 goto out_fail;
1630         }
1631
1632         vport = lport->vport;
1633
1634         if (unlikely(!hw_queue_handle)) {
1635                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1636                                  "6117 Fail IO, NULL hw_queue_handle\n");
1637                 atomic_inc(&lport->xmt_fcp_err);
1638                 ret = -EBUSY;
1639                 goto out_fail;
1640         }
1641
1642         phba = vport->phba;
1643
1644         if (unlikely(vport->load_flag & FC_UNLOADING)) {
1645                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1646                                  "6124 Fail IO, Driver unload\n");
1647                 atomic_inc(&lport->xmt_fcp_err);
1648                 ret = -ENODEV;
1649                 goto out_fail;
1650         }
1651
1652         freqpriv = pnvme_fcreq->private;
1653         if (unlikely(!freqpriv)) {
1654                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1655                                  "6158 Fail IO, NULL request data\n");
1656                 atomic_inc(&lport->xmt_fcp_err);
1657                 ret = -EINVAL;
1658                 goto out_fail;
1659         }
1660
1661 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1662         if (phba->ktime_on)
1663                 start = ktime_get_ns();
1664 #endif
1665         rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1666         lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1667
1668         /*
1669          * Catch race where our node has transitioned, but the
1670          * transport is still transitioning.
1671          */
1672         ndlp = rport->ndlp;
1673         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1674                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1675                                  "6053 Busy IO, ndlp not ready: rport x%px "
1676                                   "ndlp x%px, DID x%06x\n",
1677                                  rport, ndlp, pnvme_rport->port_id);
1678                 atomic_inc(&lport->xmt_fcp_err);
1679                 ret = -EBUSY;
1680                 goto out_fail;
1681         }
1682
1683         /* The remote node has to be a mapped target or it's an error. */
1684         if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1685             (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1686                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1687                                  "6036 Fail IO, DID x%06x not ready for "
1688                                  "IO. State x%x, Type x%x Flg x%x\n",
1689                                  pnvme_rport->port_id,
1690                                  ndlp->nlp_state, ndlp->nlp_type,
1691                                  ndlp->upcall_flags);
1692                 atomic_inc(&lport->xmt_fcp_bad_ndlp);
1693                 ret = -EBUSY;
1694                 goto out_fail;
1695
1696         }
1697
1698         /* Currently only NVME Keep alive commands should be expedited
1699          * if the driver runs out of a resource. These should only be
1700          * issued on the admin queue, qidx 0
1701          */
1702         if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1703                 sqe = &((struct nvme_fc_cmd_iu *)
1704                         pnvme_fcreq->cmdaddr)->sqe.common;
1705                 if (sqe->opcode == nvme_admin_keep_alive)
1706                         expedite = 1;
1707         }
1708
1709         /* The node is shared with FCP IO, make sure the IO pending count does
1710          * not exceed the programmed depth.
1711          */
1712         if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1713                 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1714                     !expedite) {
1715                         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1716                                          "6174 Fail IO, ndlp qdepth exceeded: "
1717                                          "idx %d DID %x pend %d qdepth %d\n",
1718                                          lpfc_queue_info->index, ndlp->nlp_DID,
1719                                          atomic_read(&ndlp->cmd_pending),
1720                                          ndlp->cmd_qdepth);
1721                         atomic_inc(&lport->xmt_fcp_qdepth);
1722                         ret = -EBUSY;
1723                         goto out_fail;
1724                 }
1725         }
1726
1727         /* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1728         if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1729                 idx = lpfc_queue_info->index;
1730         } else {
1731                 cpu = raw_smp_processor_id();
1732                 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1733         }
1734
1735         lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1736         if (lpfc_ncmd == NULL) {
1737                 atomic_inc(&lport->xmt_fcp_noxri);
1738                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1739                                  "6065 Fail IO, driver buffer pool is empty: "
1740                                  "idx %d DID %x\n",
1741                                  lpfc_queue_info->index, ndlp->nlp_DID);
1742                 ret = -EBUSY;
1743                 goto out_fail;
1744         }
1745 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1746         if (start) {
1747                 lpfc_ncmd->ts_cmd_start = start;
1748                 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1749         } else {
1750                 lpfc_ncmd->ts_cmd_start = 0;
1751         }
1752 #endif
1753
1754         /*
1755          * Store the data needed by the driver to issue, abort, and complete
1756          * an IO.
1757          * Do not let the IO hang out forever.  There is no midlayer issuing
1758          * an abort so inform the FW of the maximum IO pending time.
1759          */
1760         freqpriv->nvme_buf = lpfc_ncmd;
1761         lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1762         lpfc_ncmd->ndlp = ndlp;
1763         lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1764
1765         /*
1766          * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1767          * This identfier was create in our hardware queue create callback
1768          * routine. The driver now is dependent on the IO queue steering from
1769          * the transport.  We are trusting the upper NVME layers know which
1770          * index to use and that they have affinitized a CPU to this hardware
1771          * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1772          */
1773         lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1774         cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1775
1776         lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1777         ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1778         if (ret) {
1779                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1780                                  "6175 Fail IO, Prep DMA: "
1781                                  "idx %d DID %x\n",
1782                                  lpfc_queue_info->index, ndlp->nlp_DID);
1783                 atomic_inc(&lport->xmt_fcp_err);
1784                 ret = -ENOMEM;
1785                 goto out_free_nvme_buf;
1786         }
1787
1788         lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1789                          lpfc_ncmd->cur_iocbq.sli4_xritag,
1790                          lpfc_queue_info->index, ndlp->nlp_DID);
1791
1792         ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1793         if (ret) {
1794                 atomic_inc(&lport->xmt_fcp_wqerr);
1795                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1796                                  "6113 Fail IO, Could not issue WQE err %x "
1797                                  "sid: x%x did: x%x oxid: x%x\n",
1798                                  ret, vport->fc_myDID, ndlp->nlp_DID,
1799                                  lpfc_ncmd->cur_iocbq.sli4_xritag);
1800                 goto out_free_nvme_buf;
1801         }
1802
1803         if (phba->cfg_xri_rebalancing)
1804                 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1805
1806 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1807         if (lpfc_ncmd->ts_cmd_start)
1808                 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1809
1810         if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1811                 cpu = raw_smp_processor_id();
1812                 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1813                 lpfc_ncmd->cpu = cpu;
1814                 if (idx != cpu)
1815                         lpfc_printf_vlog(vport,
1816                                          KERN_INFO, LOG_NVME_IOERR,
1817                                         "6702 CPU Check cmd: "
1818                                         "cpu %d wq %d\n",
1819                                         lpfc_ncmd->cpu,
1820                                         lpfc_queue_info->index);
1821         }
1822 #endif
1823         return 0;
1824
1825  out_free_nvme_buf:
1826         if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1827                 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1828                         cstat->output_requests--;
1829                 else
1830                         cstat->input_requests--;
1831         } else
1832                 cstat->control_requests--;
1833         lpfc_release_nvme_buf(phba, lpfc_ncmd);
1834  out_fail:
1835         return ret;
1836 }
1837
1838 /**
1839  * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1840  * @phba: Pointer to HBA context object
1841  * @cmdiocb: Pointer to command iocb object.
1842  * @rspiocb: Pointer to response iocb object.
1843  *
1844  * This is the callback function for any NVME FCP IO that was aborted.
1845  *
1846  * Return value:
1847  *   None
1848  **/
1849 void
1850 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1851                            struct lpfc_wcqe_complete *abts_cmpl)
1852 {
1853         lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1854                         "6145 ABORT_XRI_CN completing on rpi x%x "
1855                         "original iotag x%x, abort cmd iotag x%x "
1856                         "req_tag x%x, status x%x, hwstatus x%x\n",
1857                         cmdiocb->iocb.un.acxri.abortContextTag,
1858                         cmdiocb->iocb.un.acxri.abortIoTag,
1859                         cmdiocb->iotag,
1860                         bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1861                         bf_get(lpfc_wcqe_c_status, abts_cmpl),
1862                         bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1863         lpfc_sli_release_iocbq(phba, cmdiocb);
1864 }
1865
1866 /**
1867  * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1868  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1869  * @lpfc_nvme_lport: Pointer to the driver's local port data
1870  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1871  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1872  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1873  *
1874  * Driver registers this routine as its nvme request io abort handler.  This
1875  * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1876  * data structure to the rport indicated in @lpfc_nvme_rport.  This routine
1877  * is executed asynchronously - one the target is validated as "MAPPED" and
1878  * ready for IO, the driver issues the abort request and returns.
1879  *
1880  * Return value:
1881  *   None
1882  **/
1883 static void
1884 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1885                     struct nvme_fc_remote_port *pnvme_rport,
1886                     void *hw_queue_handle,
1887                     struct nvmefc_fcp_req *pnvme_fcreq)
1888 {
1889         struct lpfc_nvme_lport *lport;
1890         struct lpfc_vport *vport;
1891         struct lpfc_hba *phba;
1892         struct lpfc_io_buf *lpfc_nbuf;
1893         struct lpfc_iocbq *abts_buf;
1894         struct lpfc_iocbq *nvmereq_wqe;
1895         struct lpfc_nvme_fcpreq_priv *freqpriv;
1896         unsigned long flags;
1897         int ret_val;
1898
1899         /* Validate pointers. LLDD fault handling with transport does
1900          * have timing races.
1901          */
1902         lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1903         if (unlikely(!lport))
1904                 return;
1905
1906         vport = lport->vport;
1907
1908         if (unlikely(!hw_queue_handle)) {
1909                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1910                                  "6129 Fail Abort, HW Queue Handle NULL.\n");
1911                 return;
1912         }
1913
1914         phba = vport->phba;
1915         freqpriv = pnvme_fcreq->private;
1916
1917         if (unlikely(!freqpriv))
1918                 return;
1919         if (vport->load_flag & FC_UNLOADING)
1920                 return;
1921
1922         /* Announce entry to new IO submit field. */
1923         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1924                          "6002 Abort Request to rport DID x%06x "
1925                          "for nvme_fc_req x%px\n",
1926                          pnvme_rport->port_id,
1927                          pnvme_fcreq);
1928
1929         /* If the hba is getting reset, this flag is set.  It is
1930          * cleared when the reset is complete and rings reestablished.
1931          */
1932         spin_lock_irqsave(&phba->hbalock, flags);
1933         /* driver queued commands are in process of being flushed */
1934         if (phba->hba_flag & HBA_IOQ_FLUSH) {
1935                 spin_unlock_irqrestore(&phba->hbalock, flags);
1936                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1937                                  "6139 Driver in reset cleanup - flushing "
1938                                  "NVME Req now.  hba_flag x%x\n",
1939                                  phba->hba_flag);
1940                 return;
1941         }
1942
1943         lpfc_nbuf = freqpriv->nvme_buf;
1944         if (!lpfc_nbuf) {
1945                 spin_unlock_irqrestore(&phba->hbalock, flags);
1946                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1947                                  "6140 NVME IO req has no matching lpfc nvme "
1948                                  "io buffer.  Skipping abort req.\n");
1949                 return;
1950         } else if (!lpfc_nbuf->nvmeCmd) {
1951                 spin_unlock_irqrestore(&phba->hbalock, flags);
1952                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1953                                  "6141 lpfc NVME IO req has no nvme_fcreq "
1954                                  "io buffer.  Skipping abort req.\n");
1955                 return;
1956         }
1957         nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1958
1959         /* Guard against IO completion being called at same time */
1960         spin_lock(&lpfc_nbuf->buf_lock);
1961
1962         /*
1963          * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1964          * state must match the nvme_fcreq passed by the nvme
1965          * transport.  If they don't match, it is likely the driver
1966          * has already completed the NVME IO and the nvme transport
1967          * has not seen it yet.
1968          */
1969         if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1970                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1971                                  "6143 NVME req mismatch: "
1972                                  "lpfc_nbuf x%px nvmeCmd x%px, "
1973                                  "pnvme_fcreq x%px.  Skipping Abort xri x%x\n",
1974                                  lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1975                                  pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1976                 goto out_unlock;
1977         }
1978
1979         /* Don't abort IOs no longer on the pending queue. */
1980         if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
1981                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1982                                  "6142 NVME IO req x%px not queued - skipping "
1983                                  "abort req xri x%x\n",
1984                                  pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1985                 goto out_unlock;
1986         }
1987
1988         atomic_inc(&lport->xmt_fcp_abort);
1989         lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1990                          nvmereq_wqe->sli4_xritag,
1991                          nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1992
1993         /* Outstanding abort is in progress */
1994         if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
1995                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1996                                  "6144 Outstanding NVME I/O Abort Request "
1997                                  "still pending on nvme_fcreq x%px, "
1998                                  "lpfc_ncmd %px xri x%x\n",
1999                                  pnvme_fcreq, lpfc_nbuf,
2000                                  nvmereq_wqe->sli4_xritag);
2001                 goto out_unlock;
2002         }
2003
2004         abts_buf = __lpfc_sli_get_iocbq(phba);
2005         if (!abts_buf) {
2006                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2007                                  "6136 No available abort wqes. Skipping "
2008                                  "Abts req for nvme_fcreq x%px xri x%x\n",
2009                                  pnvme_fcreq, nvmereq_wqe->sli4_xritag);
2010                 goto out_unlock;
2011         }
2012
2013         /* Ready - mark outstanding as aborted by driver. */
2014         nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
2015
2016         lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0);
2017
2018         /* ABTS WQE must go to the same WQ as the WQE to be aborted */
2019         abts_buf->iocb_flag |= LPFC_IO_NVME;
2020         abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx;
2021         abts_buf->vport = vport;
2022         abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
2023         ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf);
2024         spin_unlock(&lpfc_nbuf->buf_lock);
2025         spin_unlock_irqrestore(&phba->hbalock, flags);
2026         if (ret_val) {
2027                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2028                                  "6137 Failed abts issue_wqe with status x%x "
2029                                  "for nvme_fcreq x%px.\n",
2030                                  ret_val, pnvme_fcreq);
2031                 lpfc_sli_release_iocbq(phba, abts_buf);
2032                 return;
2033         }
2034
2035         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
2036                          "6138 Transport Abort NVME Request Issued for "
2037                          "ox_id x%x on reqtag x%x\n",
2038                          nvmereq_wqe->sli4_xritag,
2039                          abts_buf->iotag);
2040         return;
2041
2042 out_unlock:
2043         spin_unlock(&lpfc_nbuf->buf_lock);
2044         spin_unlock_irqrestore(&phba->hbalock, flags);
2045         return;
2046 }
2047
2048 /* Declare and initialization an instance of the FC NVME template. */
2049 static struct nvme_fc_port_template lpfc_nvme_template = {
2050         /* initiator-based functions */
2051         .localport_delete  = lpfc_nvme_localport_delete,
2052         .remoteport_delete = lpfc_nvme_remoteport_delete,
2053         .create_queue = lpfc_nvme_create_queue,
2054         .delete_queue = lpfc_nvme_delete_queue,
2055         .ls_req       = lpfc_nvme_ls_req,
2056         .fcp_io       = lpfc_nvme_fcp_io_submit,
2057         .ls_abort     = lpfc_nvme_ls_abort,
2058         .fcp_abort    = lpfc_nvme_fcp_abort,
2059         .xmt_ls_rsp   = lpfc_nvme_xmt_ls_rsp,
2060
2061         .max_hw_queues = 1,
2062         .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2063         .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2064         .dma_boundary = 0xFFFFFFFF,
2065
2066         /* Sizes of additional private data for data structures.
2067          * No use for the last two sizes at this time.
2068          */
2069         .local_priv_sz = sizeof(struct lpfc_nvme_lport),
2070         .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2071         .lsrqst_priv_sz = 0,
2072         .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2073 };
2074
2075 /**
2076  * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2077  * @phba: The HBA for which this call is being executed.
2078  *
2079  * This routine removes a nvme buffer from head of @hdwq io_buf_list
2080  * and returns to caller.
2081  *
2082  * Return codes:
2083  *   NULL - Error
2084  *   Pointer to lpfc_nvme_buf - Success
2085  **/
2086 static struct lpfc_io_buf *
2087 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2088                   int idx, int expedite)
2089 {
2090         struct lpfc_io_buf *lpfc_ncmd;
2091         struct lpfc_sli4_hdw_queue *qp;
2092         struct sli4_sge *sgl;
2093         struct lpfc_iocbq *pwqeq;
2094         union lpfc_wqe128 *wqe;
2095
2096         lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2097
2098         if (lpfc_ncmd) {
2099                 pwqeq = &(lpfc_ncmd->cur_iocbq);
2100                 wqe = &pwqeq->wqe;
2101
2102                 /* Setup key fields in buffer that may have been changed
2103                  * if other protocols used this buffer.
2104                  */
2105                 pwqeq->iocb_flag = LPFC_IO_NVME;
2106                 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
2107                 lpfc_ncmd->start_time = jiffies;
2108                 lpfc_ncmd->flags = 0;
2109
2110                 /* Rsp SGE will be filled in when we rcv an IO
2111                  * from the NVME Layer to be sent.
2112                  * The cmd is going to be embedded so we need a SKIP SGE.
2113                  */
2114                 sgl = lpfc_ncmd->dma_sgl;
2115                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2116                 bf_set(lpfc_sli4_sge_last, sgl, 0);
2117                 sgl->word2 = cpu_to_le32(sgl->word2);
2118                 /* Fill in word 3 / sgl_len during cmd submission */
2119
2120                 /* Initialize 64 bytes only */
2121                 memset(wqe, 0, sizeof(union lpfc_wqe));
2122
2123                 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2124                         atomic_inc(&ndlp->cmd_pending);
2125                         lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2126                 }
2127
2128         } else {
2129                 qp = &phba->sli4_hba.hdwq[idx];
2130                 qp->empty_io_bufs++;
2131         }
2132
2133         return  lpfc_ncmd;
2134 }
2135
2136 /**
2137  * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2138  * @phba: The Hba for which this call is being executed.
2139  * @lpfc_ncmd: The nvme buffer which is being released.
2140  *
2141  * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2142  * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2143  * and cannot be reused for at least RA_TOV amount of time if it was
2144  * aborted.
2145  **/
2146 static void
2147 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2148 {
2149         struct lpfc_sli4_hdw_queue *qp;
2150         unsigned long iflag = 0;
2151
2152         if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2153                 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2154
2155         lpfc_ncmd->ndlp = NULL;
2156         lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2157
2158         qp = lpfc_ncmd->hdwq;
2159         if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2160                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2161                                 "6310 XB release deferred for "
2162                                 "ox_id x%x on reqtag x%x\n",
2163                                 lpfc_ncmd->cur_iocbq.sli4_xritag,
2164                                 lpfc_ncmd->cur_iocbq.iotag);
2165
2166                 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2167                 list_add_tail(&lpfc_ncmd->list,
2168                         &qp->lpfc_abts_io_buf_list);
2169                 qp->abts_nvme_io_bufs++;
2170                 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2171         } else
2172                 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2173 }
2174
2175 /**
2176  * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2177  * @pvport - the lpfc_vport instance requesting a localport.
2178  *
2179  * This routine is invoked to create an nvme localport instance to bind
2180  * to the nvme_fc_transport.  It is called once during driver load
2181  * like lpfc_create_shost after all other services are initialized.
2182  * It requires a vport, vpi, and wwns at call time.  Other localport
2183  * parameters are modified as the driver's FCID and the Fabric WWN
2184  * are established.
2185  *
2186  * Return codes
2187  *      0 - successful
2188  *      -ENOMEM - no heap memory available
2189  *      other values - from nvme registration upcall
2190  **/
2191 int
2192 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2193 {
2194         int ret = 0;
2195         struct lpfc_hba  *phba = vport->phba;
2196         struct nvme_fc_port_info nfcp_info;
2197         struct nvme_fc_local_port *localport;
2198         struct lpfc_nvme_lport *lport;
2199
2200         /* Initialize this localport instance.  The vport wwn usage ensures
2201          * that NPIV is accounted for.
2202          */
2203         memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2204         nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2205         nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2206         nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2207
2208         /* We need to tell the transport layer + 1 because it takes page
2209          * alignment into account. When space for the SGL is allocated we
2210          * allocate + 3, one for cmd, one for rsp and one for this alignment
2211          */
2212         lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2213
2214         /* Advertise how many hw queues we support based on cfg_hdw_queue,
2215          * which will not exceed cpu count.
2216          */
2217         lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2218
2219         if (!IS_ENABLED(CONFIG_NVME_FC))
2220                 return ret;
2221
2222         /* localport is allocated from the stack, but the registration
2223          * call allocates heap memory as well as the private area.
2224          */
2225
2226         ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2227                                          &vport->phba->pcidev->dev, &localport);
2228         if (!ret) {
2229                 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2230                                  "6005 Successfully registered local "
2231                                  "NVME port num %d, localP x%px, private "
2232                                  "x%px, sg_seg %d\n",
2233                                  localport->port_num, localport,
2234                                  localport->private,
2235                                  lpfc_nvme_template.max_sgl_segments);
2236
2237                 /* Private is our lport size declared in the template. */
2238                 lport = (struct lpfc_nvme_lport *)localport->private;
2239                 vport->localport = localport;
2240                 lport->vport = vport;
2241                 vport->nvmei_support = 1;
2242
2243                 atomic_set(&lport->xmt_fcp_noxri, 0);
2244                 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2245                 atomic_set(&lport->xmt_fcp_qdepth, 0);
2246                 atomic_set(&lport->xmt_fcp_err, 0);
2247                 atomic_set(&lport->xmt_fcp_wqerr, 0);
2248                 atomic_set(&lport->xmt_fcp_abort, 0);
2249                 atomic_set(&lport->xmt_ls_abort, 0);
2250                 atomic_set(&lport->xmt_ls_err, 0);
2251                 atomic_set(&lport->cmpl_fcp_xb, 0);
2252                 atomic_set(&lport->cmpl_fcp_err, 0);
2253                 atomic_set(&lport->cmpl_ls_xb, 0);
2254                 atomic_set(&lport->cmpl_ls_err, 0);
2255
2256                 atomic_set(&lport->fc4NvmeLsRequests, 0);
2257                 atomic_set(&lport->fc4NvmeLsCmpls, 0);
2258         }
2259
2260         return ret;
2261 }
2262
2263 #if (IS_ENABLED(CONFIG_NVME_FC))
2264 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2265  *
2266  * The driver has to wait for the host nvme transport to callback
2267  * indicating the localport has successfully unregistered all
2268  * resources.  Since this is an uninterruptible wait, loop every ten
2269  * seconds and print a message indicating no progress.
2270  *
2271  * An uninterruptible wait is used because of the risk of transport-to-
2272  * driver state mismatch.
2273  */
2274 static void
2275 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2276                            struct lpfc_nvme_lport *lport,
2277                            struct completion *lport_unreg_cmp)
2278 {
2279         u32 wait_tmo;
2280         int ret, i, pending = 0;
2281         struct lpfc_sli_ring  *pring;
2282         struct lpfc_hba  *phba = vport->phba;
2283
2284         /* Host transport has to clean up and confirm requiring an indefinite
2285          * wait. Print a message if a 10 second wait expires and renew the
2286          * wait. This is unexpected.
2287          */
2288         wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2289         while (true) {
2290                 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2291                 if (unlikely(!ret)) {
2292                         pending = 0;
2293                         for (i = 0; i < phba->cfg_hdw_queue; i++) {
2294                                 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2295                                 if (!pring)
2296                                         continue;
2297                                 if (pring->txcmplq_cnt)
2298                                         pending += pring->txcmplq_cnt;
2299                         }
2300                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2301                                          "6176 Lport x%px Localport x%px wait "
2302                                          "timed out. Pending %d. Renewing.\n",
2303                                          lport, vport->localport, pending);
2304                         continue;
2305                 }
2306                 break;
2307         }
2308         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2309                          "6177 Lport x%px Localport x%px Complete Success\n",
2310                          lport, vport->localport);
2311 }
2312 #endif
2313
2314 /**
2315  * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2316  * @pnvme: pointer to lpfc nvme data structure.
2317  *
2318  * This routine is invoked to destroy all lports bound to the phba.
2319  * The lport memory was allocated by the nvme fc transport and is
2320  * released there.  This routine ensures all rports bound to the
2321  * lport have been disconnected.
2322  *
2323  **/
2324 void
2325 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2326 {
2327 #if (IS_ENABLED(CONFIG_NVME_FC))
2328         struct nvme_fc_local_port *localport;
2329         struct lpfc_nvme_lport *lport;
2330         int ret;
2331         DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2332
2333         if (vport->nvmei_support == 0)
2334                 return;
2335
2336         localport = vport->localport;
2337         lport = (struct lpfc_nvme_lport *)localport->private;
2338
2339         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2340                          "6011 Destroying NVME localport x%px\n",
2341                          localport);
2342
2343         /* lport's rport list is clear.  Unregister
2344          * lport and release resources.
2345          */
2346         lport->lport_unreg_cmp = &lport_unreg_cmp;
2347         ret = nvme_fc_unregister_localport(localport);
2348
2349         /* Wait for completion.  This either blocks
2350          * indefinitely or succeeds
2351          */
2352         lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2353         vport->localport = NULL;
2354
2355         /* Regardless of the unregister upcall response, clear
2356          * nvmei_support.  All rports are unregistered and the
2357          * driver will clean up.
2358          */
2359         vport->nvmei_support = 0;
2360         if (ret == 0) {
2361                 lpfc_printf_vlog(vport,
2362                                  KERN_INFO, LOG_NVME_DISC,
2363                                  "6009 Unregistered lport Success\n");
2364         } else {
2365                 lpfc_printf_vlog(vport,
2366                                  KERN_INFO, LOG_NVME_DISC,
2367                                  "6010 Unregistered lport "
2368                                  "Failed, status x%x\n",
2369                                  ret);
2370         }
2371 #endif
2372 }
2373
2374 void
2375 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2376 {
2377 #if (IS_ENABLED(CONFIG_NVME_FC))
2378         struct nvme_fc_local_port *localport;
2379         struct lpfc_nvme_lport *lport;
2380
2381         localport = vport->localport;
2382         if (!localport) {
2383                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2384                                  "6710 Update NVME fail. No localport\n");
2385                 return;
2386         }
2387         lport = (struct lpfc_nvme_lport *)localport->private;
2388         if (!lport) {
2389                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2390                                  "6171 Update NVME fail. localP x%px, No lport\n",
2391                                  localport);
2392                 return;
2393         }
2394         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2395                          "6012 Update NVME lport x%px did x%x\n",
2396                          localport, vport->fc_myDID);
2397
2398         localport->port_id = vport->fc_myDID;
2399         if (localport->port_id == 0)
2400                 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2401         else
2402                 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2403
2404         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2405                          "6030 bound lport x%px to DID x%06x\n",
2406                          lport, localport->port_id);
2407 #endif
2408 }
2409
2410 int
2411 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2412 {
2413 #if (IS_ENABLED(CONFIG_NVME_FC))
2414         int ret = 0;
2415         struct nvme_fc_local_port *localport;
2416         struct lpfc_nvme_lport *lport;
2417         struct lpfc_nvme_rport *rport;
2418         struct lpfc_nvme_rport *oldrport;
2419         struct nvme_fc_remote_port *remote_port;
2420         struct nvme_fc_port_info rpinfo;
2421         struct lpfc_nodelist *prev_ndlp = NULL;
2422         struct fc_rport *srport = ndlp->rport;
2423
2424         lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2425                          "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2426                          ndlp->nlp_DID, ndlp->nlp_type);
2427
2428         localport = vport->localport;
2429         if (!localport)
2430                 return 0;
2431
2432         lport = (struct lpfc_nvme_lport *)localport->private;
2433
2434         /* NVME rports are not preserved across devloss.
2435          * Just register this instance.  Note, rpinfo->dev_loss_tmo
2436          * is left 0 to indicate accept transport defaults.  The
2437          * driver communicates port role capabilities consistent
2438          * with the PRLI response data.
2439          */
2440         memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2441         rpinfo.port_id = ndlp->nlp_DID;
2442         if (ndlp->nlp_type & NLP_NVME_TARGET)
2443                 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2444         if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2445                 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2446
2447         if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2448                 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2449
2450         rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2451         rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2452         if (srport)
2453                 rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2454         else
2455                 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2456
2457         spin_lock_irq(&vport->phba->hbalock);
2458         oldrport = lpfc_ndlp_get_nrport(ndlp);
2459         if (oldrport) {
2460                 prev_ndlp = oldrport->ndlp;
2461                 spin_unlock_irq(&vport->phba->hbalock);
2462         } else {
2463                 spin_unlock_irq(&vport->phba->hbalock);
2464                 lpfc_nlp_get(ndlp);
2465         }
2466
2467         ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2468         if (!ret) {
2469                 /* If the ndlp already has an nrport, this is just
2470                  * a resume of the existing rport.  Else this is a
2471                  * new rport.
2472                  */
2473                 /* Guard against an unregister/reregister
2474                  * race that leaves the WAIT flag set.
2475                  */
2476                 spin_lock_irq(&vport->phba->hbalock);
2477                 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2478                 spin_unlock_irq(&vport->phba->hbalock);
2479                 rport = remote_port->private;
2480                 if (oldrport) {
2481
2482                         /* Sever the ndlp<->rport association
2483                          * before dropping the ndlp ref from
2484                          * register.
2485                          */
2486                         spin_lock_irq(&vport->phba->hbalock);
2487                         ndlp->nrport = NULL;
2488                         ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2489                         spin_unlock_irq(&vport->phba->hbalock);
2490                         rport->ndlp = NULL;
2491                         rport->remoteport = NULL;
2492
2493                         /* Reference only removed if previous NDLP is no longer
2494                          * active. It might be just a swap and removing the
2495                          * reference would cause a premature cleanup.
2496                          */
2497                         if (prev_ndlp && prev_ndlp != ndlp) {
2498                                 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
2499                                     (!prev_ndlp->nrport))
2500                                         lpfc_nlp_put(prev_ndlp);
2501                         }
2502                 }
2503
2504                 /* Clean bind the rport to the ndlp. */
2505                 rport->remoteport = remote_port;
2506                 rport->lport = lport;
2507                 rport->ndlp = ndlp;
2508                 spin_lock_irq(&vport->phba->hbalock);
2509                 ndlp->nrport = rport;
2510                 spin_unlock_irq(&vport->phba->hbalock);
2511                 lpfc_printf_vlog(vport, KERN_INFO,
2512                                  LOG_NVME_DISC | LOG_NODE,
2513                                  "6022 Bind lport x%px to remoteport x%px "
2514                                  "rport x%px WWNN 0x%llx, "
2515                                  "Rport WWPN 0x%llx DID "
2516                                  "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2517                                  lport, remote_port, rport,
2518                                  rpinfo.node_name, rpinfo.port_name,
2519                                  rpinfo.port_id, rpinfo.port_role,
2520                                  ndlp, prev_ndlp);
2521         } else {
2522                 lpfc_printf_vlog(vport, KERN_ERR,
2523                                  LOG_TRACE_EVENT,
2524                                  "6031 RemotePort Registration failed "
2525                                  "err: %d, DID x%06x\n",
2526                                  ret, ndlp->nlp_DID);
2527         }
2528
2529         return ret;
2530 #else
2531         return 0;
2532 #endif
2533 }
2534
2535 /**
2536  * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2537  *
2538  * If the ndlp represents an NVME Target, that we are logged into,
2539  * ping the NVME FC Transport layer to initiate a device rescan
2540  * on this remote NPort.
2541  */
2542 void
2543 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2544 {
2545 #if (IS_ENABLED(CONFIG_NVME_FC))
2546         struct lpfc_nvme_rport *nrport;
2547         struct nvme_fc_remote_port *remoteport = NULL;
2548
2549         spin_lock_irq(&vport->phba->hbalock);
2550         nrport = lpfc_ndlp_get_nrport(ndlp);
2551         if (nrport)
2552                 remoteport = nrport->remoteport;
2553         spin_unlock_irq(&vport->phba->hbalock);
2554
2555         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2556                          "6170 Rescan NPort DID x%06x type x%x "
2557                          "state x%x nrport x%px remoteport x%px\n",
2558                          ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2559                          nrport, remoteport);
2560
2561         if (!nrport || !remoteport)
2562                 goto rescan_exit;
2563
2564         /* Only rescan if we are an NVME target in the MAPPED state */
2565         if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2566             ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2567                 nvme_fc_rescan_remoteport(remoteport);
2568
2569                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2570                                  "6172 NVME rescanned DID x%06x "
2571                                  "port_state x%x\n",
2572                                  ndlp->nlp_DID, remoteport->port_state);
2573         }
2574         return;
2575  rescan_exit:
2576         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2577                          "6169 Skip NVME Rport Rescan, NVME remoteport "
2578                          "unregistered\n");
2579 #endif
2580 }
2581
2582 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2583  *
2584  * There is no notion of Devloss or rport recovery from the current
2585  * nvme_transport perspective.  Loss of an rport just means IO cannot
2586  * be sent and recovery is completely up to the initator.
2587  * For now, the driver just unbinds the DID and port_role so that
2588  * no further IO can be issued.  Changes are planned for later.
2589  *
2590  * Notes - the ndlp reference count is not decremented here since
2591  * since there is no nvme_transport api for devloss.  Node ref count
2592  * is only adjusted in driver unload.
2593  */
2594 void
2595 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2596 {
2597 #if (IS_ENABLED(CONFIG_NVME_FC))
2598         int ret;
2599         struct nvme_fc_local_port *localport;
2600         struct lpfc_nvme_lport *lport;
2601         struct lpfc_nvme_rport *rport;
2602         struct nvme_fc_remote_port *remoteport = NULL;
2603
2604         localport = vport->localport;
2605
2606         /* This is fundamental error.  The localport is always
2607          * available until driver unload.  Just exit.
2608          */
2609         if (!localport)
2610                 return;
2611
2612         lport = (struct lpfc_nvme_lport *)localport->private;
2613         if (!lport)
2614                 goto input_err;
2615
2616         spin_lock_irq(&vport->phba->hbalock);
2617         rport = lpfc_ndlp_get_nrport(ndlp);
2618         if (rport)
2619                 remoteport = rport->remoteport;
2620         spin_unlock_irq(&vport->phba->hbalock);
2621         if (!remoteport)
2622                 goto input_err;
2623
2624         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2625                          "6033 Unreg nvme remoteport x%px, portname x%llx, "
2626                          "port_id x%06x, portstate x%x port type x%x\n",
2627                          remoteport, remoteport->port_name,
2628                          remoteport->port_id, remoteport->port_state,
2629                          ndlp->nlp_type);
2630
2631         /* Sanity check ndlp type.  Only call for NVME ports. Don't
2632          * clear any rport state until the transport calls back.
2633          */
2634
2635         if (ndlp->nlp_type & NLP_NVME_TARGET) {
2636                 /* No concern about the role change on the nvme remoteport.
2637                  * The transport will update it.
2638                  */
2639                 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
2640
2641                 /* Don't let the host nvme transport keep sending keep-alives
2642                  * on this remoteport. Vport is unloading, no recovery. The
2643                  * return values is ignored.  The upcall is a courtesy to the
2644                  * transport.
2645                  */
2646                 if (vport->load_flag & FC_UNLOADING)
2647                         (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2648
2649                 ret = nvme_fc_unregister_remoteport(remoteport);
2650                 if (ret != 0) {
2651                         lpfc_nlp_put(ndlp);
2652                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2653                                          "6167 NVME unregister failed %d "
2654                                          "port_state x%x\n",
2655                                          ret, remoteport->port_state);
2656                 }
2657         }
2658         return;
2659
2660  input_err:
2661 #endif
2662         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2663                          "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2664                          vport->localport, ndlp->rport, ndlp->nlp_DID);
2665 }
2666
2667 /**
2668  * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2669  * @phba: pointer to lpfc hba data structure.
2670  * @axri: pointer to the fcp xri abort wcqe structure.
2671  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2672  *
2673  * This routine is invoked by the worker thread to process a SLI4 fast-path
2674  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2675  * here.
2676  **/
2677 void
2678 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2679                            struct sli4_wcqe_xri_aborted *axri,
2680                            struct lpfc_io_buf *lpfc_ncmd)
2681 {
2682         uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2683         struct nvmefc_fcp_req *nvme_cmd = NULL;
2684         struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2685
2686
2687         if (ndlp)
2688                 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2689
2690         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2691                         "6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2692                         "xri released\n",
2693                         lpfc_ncmd->nvmeCmd, xri,
2694                         lpfc_ncmd->cur_iocbq.iotag);
2695
2696         /* Aborted NVME commands are required to not complete
2697          * before the abort exchange command fully completes.
2698          * Once completed, it is available via the put list.
2699          */
2700         if (lpfc_ncmd->nvmeCmd) {
2701                 nvme_cmd = lpfc_ncmd->nvmeCmd;
2702                 nvme_cmd->done(nvme_cmd);
2703                 lpfc_ncmd->nvmeCmd = NULL;
2704         }
2705         lpfc_release_nvme_buf(phba, lpfc_ncmd);
2706 }
2707
2708 /**
2709  * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2710  * @phba: Pointer to HBA context object.
2711  *
2712  * This function flushes all wqes in the nvme rings and frees all resources
2713  * in the txcmplq. This function does not issue abort wqes for the IO
2714  * commands in txcmplq, they will just be returned with
2715  * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2716  * slot has been permanently disabled.
2717  **/
2718 void
2719 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2720 {
2721         struct lpfc_sli_ring  *pring;
2722         u32 i, wait_cnt = 0;
2723
2724         if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2725                 return;
2726
2727         /* Cycle through all IO rings and make sure all outstanding
2728          * WQEs have been removed from the txcmplqs.
2729          */
2730         for (i = 0; i < phba->cfg_hdw_queue; i++) {
2731                 if (!phba->sli4_hba.hdwq[i].io_wq)
2732                         continue;
2733                 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2734
2735                 if (!pring)
2736                         continue;
2737
2738                 /* Retrieve everything on the txcmplq */
2739                 while (!list_empty(&pring->txcmplq)) {
2740                         msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2741                         wait_cnt++;
2742
2743                         /* The sleep is 10mS.  Every ten seconds,
2744                          * dump a message.  Something is wrong.
2745                          */
2746                         if ((wait_cnt % 1000) == 0) {
2747                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2748                                                 "6178 NVME IO not empty, "
2749                                                 "cnt %d\n", wait_cnt);
2750                         }
2751                 }
2752         }
2753 }
2754
2755 void
2756 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn)
2757 {
2758 #if (IS_ENABLED(CONFIG_NVME_FC))
2759         struct lpfc_io_buf *lpfc_ncmd;
2760         struct nvmefc_fcp_req *nCmd;
2761         struct lpfc_nvme_fcpreq_priv *freqpriv;
2762
2763         if (!pwqeIn->context1) {
2764                 lpfc_sli_release_iocbq(phba, pwqeIn);
2765                 return;
2766         }
2767         /* For abort iocb just return, IO iocb will do a done call */
2768         if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2769             CMD_ABORT_XRI_CX) {
2770                 lpfc_sli_release_iocbq(phba, pwqeIn);
2771                 return;
2772         }
2773         lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1;
2774
2775         spin_lock(&lpfc_ncmd->buf_lock);
2776         if (!lpfc_ncmd->nvmeCmd) {
2777                 spin_unlock(&lpfc_ncmd->buf_lock);
2778                 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2779                 return;
2780         }
2781
2782         nCmd = lpfc_ncmd->nvmeCmd;
2783         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2784                         "6194 NVME Cancel xri %x\n",
2785                         lpfc_ncmd->cur_iocbq.sli4_xritag);
2786
2787         nCmd->transferred_length = 0;
2788         nCmd->rcv_rsplen = 0;
2789         nCmd->status = NVME_SC_INTERNAL;
2790         freqpriv = nCmd->private;
2791         freqpriv->nvme_buf = NULL;
2792         lpfc_ncmd->nvmeCmd = NULL;
2793
2794         spin_unlock(&lpfc_ncmd->buf_lock);
2795         nCmd->done(nCmd);
2796
2797         /* Call release with XB=1 to queue the IO into the abort list. */
2798         lpfc_release_nvme_buf(phba, lpfc_ncmd);
2799 #endif
2800 }