drivers/scsi/qedf/qedf_io.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  QLogic FCoE Offload Driver
   4  *  Copyright (c) 2016-2018 Cavium Inc.
   5  */
   6 #include <linux/spinlock.h>
   7 #include <linux/vmalloc.h>
   8 #include "qedf.h"
   9 #include <scsi/scsi_tcq.h>
  10
  11 void qedf_cmd_timer_set(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
  12         unsigned int timer_msec)
  13 {
  14         queue_delayed_work(qedf->timer_work_queue, &io_req->timeout_work,
  15             msecs_to_jiffies(timer_msec));
  16 }
  17
  18 static void qedf_cmd_timeout(struct work_struct *work)
  19 {
  20
  21         struct qedf_ioreq *io_req =
  22             container_of(work, struct qedf_ioreq, timeout_work.work);
  23         struct qedf_ctx *qedf;
  24         struct qedf_rport *fcport;
  25
  26         if (io_req == NULL) {
  27                 QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
  28                 return;
  29         }
  30
  31         fcport = io_req->fcport;
  32         if (io_req->fcport == NULL) {
  33                 QEDF_INFO(NULL, QEDF_LOG_IO,  "fcport is NULL.\n");
  34                 return;
  35         }
  36
  37         qedf = fcport->qedf;
  38
  39         switch (io_req->cmd_type) {
  40         case QEDF_ABTS:
  41                 if (qedf == NULL) {
  42                         QEDF_INFO(NULL, QEDF_LOG_IO,
  43                                   "qedf is NULL for ABTS xid=0x%x.\n",
  44                                   io_req->xid);
  45                         return;
  46                 }
  47
  48                 QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
  49                     io_req->xid);
  50                 /* Cleanup timed out ABTS */
  51                 qedf_initiate_cleanup(io_req, true);
  52                 complete(&io_req->abts_done);
  53
  54                 /*
  55                  * Need to call kref_put for reference taken when initiate_abts
  56                  * was called since abts_compl won't be called now that we've
  57                  * cleaned up the task.
  58                  */
  59                 kref_put(&io_req->refcount, qedf_release_cmd);
  60
  61                 /* Clear in abort bit now that we're done with the command */
  62                 clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
  63
  64                 /*
  65                  * Now that the original I/O and the ABTS are complete see
  66                  * if we need to reconnect to the target.
  67                  */
  68                 qedf_restart_rport(fcport);
  69                 break;
  70         case QEDF_ELS:
  71                 if (!qedf) {
  72                         QEDF_INFO(NULL, QEDF_LOG_IO,
  73                                   "qedf is NULL for ELS xid=0x%x.\n",
  74                                   io_req->xid);
  75                         return;
  76                 }
  77                 /* ELS request no longer outstanding since it timed out */
  78                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
  79
  80                 kref_get(&io_req->refcount);
  81                 /*
  82                  * Don't attempt to clean an ELS timeout as any subseqeunt
  83                  * ABTS or cleanup requests just hang.  For now just free
  84                  * the resources of the original I/O and the RRQ
  85                  */
  86                 QEDF_ERR(&(qedf->dbg_ctx), "ELS timeout, xid=0x%x.\n",
  87                           io_req->xid);
  88                 qedf_initiate_cleanup(io_req, true);
  89                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
  90                 /* Call callback function to complete command */
  91                 if (io_req->cb_func && io_req->cb_arg) {
  92                         io_req->cb_func(io_req->cb_arg);
  93                         io_req->cb_arg = NULL;
  94                 }
  95                 kref_put(&io_req->refcount, qedf_release_cmd);
  96                 break;
  97         case QEDF_SEQ_CLEANUP:
  98                 QEDF_ERR(&(qedf->dbg_ctx), "Sequence cleanup timeout, "
  99                     "xid=0x%x.\n", io_req->xid);
 100                 qedf_initiate_cleanup(io_req, true);
 101                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
 102                 qedf_process_seq_cleanup_compl(qedf, NULL, io_req);
 103                 break;
 104         default:
 105                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 106                           "Hit default case, xid=0x%x.\n", io_req->xid);
 107                 break;
 108         }
 109 }
 110
 111 void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr)
 112 {
 113         struct io_bdt *bdt_info;
 114         struct qedf_ctx *qedf = cmgr->qedf;
 115         size_t bd_tbl_sz;
 116         u16 min_xid = 0;
 117         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 118         int num_ios;
 119         int i;
 120         struct qedf_ioreq *io_req;
 121
 122         num_ios = max_xid - min_xid + 1;
 123
 124         /* Free fcoe_bdt_ctx structures */
 125         if (!cmgr->io_bdt_pool) {
 126                 QEDF_ERR(&qedf->dbg_ctx, "io_bdt_pool is NULL.\n");
 127                 goto free_cmd_pool;
 128         }
 129
 130         bd_tbl_sz = QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge);
 131         for (i = 0; i < num_ios; i++) {
 132                 bdt_info = cmgr->io_bdt_pool[i];
 133                 if (bdt_info->bd_tbl) {
 134                         dma_free_coherent(&qedf->pdev->dev, bd_tbl_sz,
 135                             bdt_info->bd_tbl, bdt_info->bd_tbl_dma);
 136                         bdt_info->bd_tbl = NULL;
 137                 }
 138         }
 139
 140         /* Destroy io_bdt pool */
 141         for (i = 0; i < num_ios; i++) {
 142                 kfree(cmgr->io_bdt_pool[i]);
 143                 cmgr->io_bdt_pool[i] = NULL;
 144         }
 145
 146         kfree(cmgr->io_bdt_pool);
 147         cmgr->io_bdt_pool = NULL;
 148
 149 free_cmd_pool:
 150
 151         for (i = 0; i < num_ios; i++) {
 152                 io_req = &cmgr->cmds[i];
 153                 kfree(io_req->sgl_task_params);
 154                 kfree(io_req->task_params);
 155                 /* Make sure we free per command sense buffer */
 156                 if (io_req->sense_buffer)
 157                         dma_free_coherent(&qedf->pdev->dev,
 158                             QEDF_SCSI_SENSE_BUFFERSIZE, io_req->sense_buffer,
 159                             io_req->sense_buffer_dma);
 160                 cancel_delayed_work_sync(&io_req->rrq_work);
 161         }
 162
 163         /* Free command manager itself */
 164         vfree(cmgr);
 165 }
 166
 167 static void qedf_handle_rrq(struct work_struct *work)
 168 {
 169         struct qedf_ioreq *io_req =
 170             container_of(work, struct qedf_ioreq, rrq_work.work);
 171
 172         atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_ACTIVE);
 173         qedf_send_rrq(io_req);
 174
 175 }
 176
 177 struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
 178 {
 179         struct qedf_cmd_mgr *cmgr;
 180         struct io_bdt *bdt_info;
 181         struct qedf_ioreq *io_req;
 182         u16 xid;
 183         int i;
 184         int num_ios;
 185         u16 min_xid = 0;
 186         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 187
 188         /* Make sure num_queues is already set before calling this function */
 189         if (!qedf->num_queues) {
 190                 QEDF_ERR(&(qedf->dbg_ctx), "num_queues is not set.\n");
 191                 return NULL;
 192         }
 193
 194         if (max_xid <= min_xid || max_xid == FC_XID_UNKNOWN) {
 195                 QEDF_WARN(&(qedf->dbg_ctx), "Invalid min_xid 0x%x and "
 196                            "max_xid 0x%x.\n", min_xid, max_xid);
 197                 return NULL;
 198         }
 199
 200         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "min xid 0x%x, max xid "
 201                    "0x%x.\n", min_xid, max_xid);
 202
 203         num_ios = max_xid - min_xid + 1;
 204
 205         cmgr = vzalloc(sizeof(struct qedf_cmd_mgr));
 206         if (!cmgr) {
 207                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc cmd mgr.\n");
 208                 return NULL;
 209         }
 210
 211         cmgr->qedf = qedf;
 212         spin_lock_init(&cmgr->lock);
 213
 214         /*
 215          * Initialize I/O request fields.
 216          */
 217         xid = 0;
 218
 219         for (i = 0; i < num_ios; i++) {
 220                 io_req = &cmgr->cmds[i];
 221                 INIT_DELAYED_WORK(&io_req->timeout_work, qedf_cmd_timeout);
 222
 223                 io_req->xid = xid++;
 224
 225                 INIT_DELAYED_WORK(&io_req->rrq_work, qedf_handle_rrq);
 226
 227                 /* Allocate DMA memory to hold sense buffer */
 228                 io_req->sense_buffer = dma_alloc_coherent(&qedf->pdev->dev,
 229                     QEDF_SCSI_SENSE_BUFFERSIZE, &io_req->sense_buffer_dma,
 230                     GFP_KERNEL);
 231                 if (!io_req->sense_buffer) {
 232                         QEDF_ERR(&qedf->dbg_ctx,
 233                                  "Failed to alloc sense buffer.\n");
 234                         goto mem_err;
 235                 }
 236
 237                 /* Allocate task parameters to pass to f/w init funcions */
 238                 io_req->task_params = kzalloc(sizeof(*io_req->task_params),
 239                                               GFP_KERNEL);
 240                 if (!io_req->task_params) {
 241                         QEDF_ERR(&(qedf->dbg_ctx),
 242                                  "Failed to allocate task_params for xid=0x%x\n",
 243                                  i);
 244                         goto mem_err;
 245                 }
 246
 247                 /*
 248                  * Allocate scatter/gather list info to pass to f/w init
 249                  * functions.
 250                  */
 251                 io_req->sgl_task_params = kzalloc(
 252                     sizeof(struct scsi_sgl_task_params), GFP_KERNEL);
 253                 if (!io_req->sgl_task_params) {
 254                         QEDF_ERR(&(qedf->dbg_ctx),
 255                                  "Failed to allocate sgl_task_params for xid=0x%x\n",
 256                                  i);
 257                         goto mem_err;
 258                 }
 259         }
 260
 261         /* Allocate pool of io_bdts - one for each qedf_ioreq */
 262         cmgr->io_bdt_pool = kmalloc_array(num_ios, sizeof(struct io_bdt *),
 263             GFP_KERNEL);
 264
 265         if (!cmgr->io_bdt_pool) {
 266                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc io_bdt_pool.\n");
 267                 goto mem_err;
 268         }
 269
 270         for (i = 0; i < num_ios; i++) {
 271                 cmgr->io_bdt_pool[i] = kmalloc(sizeof(struct io_bdt),
 272                     GFP_KERNEL);
 273                 if (!cmgr->io_bdt_pool[i]) {
 274                         QEDF_WARN(&(qedf->dbg_ctx),
 275                                   "Failed to alloc io_bdt_pool[%d].\n", i);
 276                         goto mem_err;
 277                 }
 278         }
 279
 280         for (i = 0; i < num_ios; i++) {
 281                 bdt_info = cmgr->io_bdt_pool[i];
 282                 bdt_info->bd_tbl = dma_alloc_coherent(&qedf->pdev->dev,
 283                     QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge),
 284                     &bdt_info->bd_tbl_dma, GFP_KERNEL);
 285                 if (!bdt_info->bd_tbl) {
 286                         QEDF_WARN(&(qedf->dbg_ctx),
 287                                   "Failed to alloc bdt_tbl[%d].\n", i);
 288                         goto mem_err;
 289                 }
 290         }
 291         atomic_set(&cmgr->free_list_cnt, num_ios);
 292         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 293             "cmgr->free_list_cnt=%d.\n",
 294             atomic_read(&cmgr->free_list_cnt));
 295
 296         return cmgr;
 297
 298 mem_err:
 299         qedf_cmd_mgr_free(cmgr);
 300         return NULL;
 301 }
 302
 303 struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
 304 {
 305         struct qedf_ctx *qedf = fcport->qedf;
 306         struct qedf_cmd_mgr *cmd_mgr = qedf->cmd_mgr;
 307         struct qedf_ioreq *io_req = NULL;
 308         struct io_bdt *bd_tbl;
 309         u16 xid;
 310         uint32_t free_sqes;
 311         int i;
 312         unsigned long flags;
 313
 314         free_sqes = atomic_read(&fcport->free_sqes);
 315
 316         if (!free_sqes) {
 317                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 318                     "Returning NULL, free_sqes=%d.\n ",
 319                     free_sqes);
 320                 goto out_failed;
 321         }
 322
 323         /* Limit the number of outstanding R/W tasks */
 324         if ((atomic_read(&fcport->num_active_ios) >=
 325             NUM_RW_TASKS_PER_CONNECTION)) {
 326                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 327                     "Returning NULL, num_active_ios=%d.\n",
 328                     atomic_read(&fcport->num_active_ios));
 329                 goto out_failed;
 330         }
 331
 332         /* Limit global TIDs certain tasks */
 333         if (atomic_read(&cmd_mgr->free_list_cnt) <= GBL_RSVD_TASKS) {
 334                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 335                     "Returning NULL, free_list_cnt=%d.\n",
 336                     atomic_read(&cmd_mgr->free_list_cnt));
 337                 goto out_failed;
 338         }
 339
 340         spin_lock_irqsave(&cmd_mgr->lock, flags);
 341         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
 342                 io_req = &cmd_mgr->cmds[cmd_mgr->idx];
 343                 cmd_mgr->idx++;
 344                 if (cmd_mgr->idx == FCOE_PARAMS_NUM_TASKS)
 345                         cmd_mgr->idx = 0;
 346
 347                 /* Check to make sure command was previously freed */
 348                 if (!io_req->alloc)
 349                         break;
 350         }
 351
 352         if (i == FCOE_PARAMS_NUM_TASKS) {
 353                 spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 354                 goto out_failed;
 355         }
 356
 357         if (test_bit(QEDF_CMD_DIRTY, &io_req->flags))
 358                 QEDF_ERR(&qedf->dbg_ctx,
 359                          "io_req found to be dirty ox_id = 0x%x.\n",
 360                          io_req->xid);
 361
 362         /* Clear any flags now that we've reallocated the xid */
 363         io_req->flags = 0;
 364         io_req->alloc = 1;
 365         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 366
 367         atomic_inc(&fcport->num_active_ios);
 368         atomic_dec(&fcport->free_sqes);
 369         xid = io_req->xid;
 370         atomic_dec(&cmd_mgr->free_list_cnt);
 371
 372         io_req->cmd_mgr = cmd_mgr;
 373         io_req->fcport = fcport;
 374
 375         /* Clear any stale sc_cmd back pointer */
 376         io_req->sc_cmd = NULL;
 377         io_req->lun = -1;
 378
 379         /* Hold the io_req against deletion */
 380         kref_init(&io_req->refcount);   /* ID: 001 */
 381         atomic_set(&io_req->state, QEDFC_CMD_ST_IO_ACTIVE);
 382
 383         /* Bind io_bdt for this io_req */
 384         /* Have a static link between io_req and io_bdt_pool */
 385         bd_tbl = io_req->bd_tbl = cmd_mgr->io_bdt_pool[xid];
 386         if (bd_tbl == NULL) {
 387                 QEDF_ERR(&(qedf->dbg_ctx), "bd_tbl is NULL, xid=%x.\n", xid);
 388                 kref_put(&io_req->refcount, qedf_release_cmd);
 389                 goto out_failed;
 390         }
 391         bd_tbl->io_req = io_req;
 392         io_req->cmd_type = cmd_type;
 393         io_req->tm_flags = 0;
 394
 395         /* Reset sequence offset data */
 396         io_req->rx_buf_off = 0;
 397         io_req->tx_buf_off = 0;
 398         io_req->rx_id = 0xffff; /* No OX_ID */
 399
 400         return io_req;
 401
 402 out_failed:
 403         /* Record failure for stats and return NULL to caller */
 404         qedf->alloc_failures++;
 405         return NULL;
 406 }
 407
 408 static void qedf_free_mp_resc(struct qedf_ioreq *io_req)
 409 {
 410         struct qedf_mp_req *mp_req = &(io_req->mp_req);
 411         struct qedf_ctx *qedf = io_req->fcport->qedf;
 412         uint64_t sz = sizeof(struct scsi_sge);
 413
 414         /* clear tm flags */
 415         if (mp_req->mp_req_bd) {
 416                 dma_free_coherent(&qedf->pdev->dev, sz,
 417                     mp_req->mp_req_bd, mp_req->mp_req_bd_dma);
 418                 mp_req->mp_req_bd = NULL;
 419         }
 420         if (mp_req->mp_resp_bd) {
 421                 dma_free_coherent(&qedf->pdev->dev, sz,
 422                     mp_req->mp_resp_bd, mp_req->mp_resp_bd_dma);
 423                 mp_req->mp_resp_bd = NULL;
 424         }
 425         if (mp_req->req_buf) {
 426                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
 427                     mp_req->req_buf, mp_req->req_buf_dma);
 428                 mp_req->req_buf = NULL;
 429         }
 430         if (mp_req->resp_buf) {
 431                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
 432                     mp_req->resp_buf, mp_req->resp_buf_dma);
 433                 mp_req->resp_buf = NULL;
 434         }
 435 }
 436
 437 void qedf_release_cmd(struct kref *ref)
 438 {
 439         struct qedf_ioreq *io_req =
 440             container_of(ref, struct qedf_ioreq, refcount);
 441         struct qedf_cmd_mgr *cmd_mgr = io_req->cmd_mgr;
 442         struct qedf_rport *fcport = io_req->fcport;
 443         unsigned long flags;
 444
 445         if (io_req->cmd_type == QEDF_SCSI_CMD) {
 446                 QEDF_WARN(&fcport->qedf->dbg_ctx,
 447                           "Cmd released called without scsi_done called, io_req %p xid=0x%x.\n",
 448                           io_req, io_req->xid);
 449                 WARN_ON(io_req->sc_cmd);
 450         }
 451
 452         if (io_req->cmd_type == QEDF_ELS ||
 453             io_req->cmd_type == QEDF_TASK_MGMT_CMD)
 454                 qedf_free_mp_resc(io_req);
 455
 456         atomic_inc(&cmd_mgr->free_list_cnt);
 457         atomic_dec(&fcport->num_active_ios);
 458         atomic_set(&io_req->state, QEDF_CMD_ST_INACTIVE);
 459         if (atomic_read(&fcport->num_active_ios) < 0) {
 460                 QEDF_WARN(&(fcport->qedf->dbg_ctx), "active_ios < 0.\n");
 461                 WARN_ON(1);
 462         }
 463
 464         /* Increment task retry identifier now that the request is released */
 465         io_req->task_retry_identifier++;
 466         io_req->fcport = NULL;
 467
 468         clear_bit(QEDF_CMD_DIRTY, &io_req->flags);
 469         io_req->cpu = 0;
 470         spin_lock_irqsave(&cmd_mgr->lock, flags);
 471         io_req->fcport = NULL;
 472         io_req->alloc = 0;
 473         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 474 }
 475
 476 static int qedf_map_sg(struct qedf_ioreq *io_req)
 477 {
 478         struct scsi_cmnd *sc = io_req->sc_cmd;
 479         struct Scsi_Host *host = sc->device->host;
 480         struct fc_lport *lport = shost_priv(host);
 481         struct qedf_ctx *qedf = lport_priv(lport);
 482         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
 483         struct scatterlist *sg;
 484         int byte_count = 0;
 485         int sg_count = 0;
 486         int bd_count = 0;
 487         u32 sg_len;
 488         u64 addr;
 489         int i = 0;
 490
 491         sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
 492             scsi_sg_count(sc), sc->sc_data_direction);
 493         sg = scsi_sglist(sc);
 494
 495         io_req->sge_type = QEDF_IOREQ_UNKNOWN_SGE;
 496
 497         if (sg_count <= 8 || io_req->io_req_flags == QEDF_READ)
 498                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 499
 500         scsi_for_each_sg(sc, sg, sg_count, i) {
 501                 sg_len = (u32)sg_dma_len(sg);
 502                 addr = (u64)sg_dma_address(sg);
 503
 504                 /*
 505                  * Intermediate s/g element so check if start address
 506                  * is page aligned.  Only required for writes and only if the
 507                  * number of scatter/gather elements is 8 or more.
 508                  */
 509                 if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE && (i) &&
 510                     (i != (sg_count - 1)) && sg_len < QEDF_PAGE_SIZE)
 511                         io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
 512
 513                 bd[bd_count].sge_addr.lo = cpu_to_le32(U64_LO(addr));
 514                 bd[bd_count].sge_addr.hi  = cpu_to_le32(U64_HI(addr));
 515                 bd[bd_count].sge_len = cpu_to_le32(sg_len);
 516
 517                 bd_count++;
 518                 byte_count += sg_len;
 519         }
 520
 521         /* To catch a case where FAST and SLOW nothing is set, set FAST */
 522         if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE)
 523                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 524
 525         if (byte_count != scsi_bufflen(sc))
 526                 QEDF_ERR(&(qedf->dbg_ctx), "byte_count = %d != "
 527                           "scsi_bufflen = %d, task_id = 0x%x.\n", byte_count,
 528                            scsi_bufflen(sc), io_req->xid);
 529
 530         return bd_count;
 531 }
 532
 533 static int qedf_build_bd_list_from_sg(struct qedf_ioreq *io_req)
 534 {
 535         struct scsi_cmnd *sc = io_req->sc_cmd;
 536         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
 537         int bd_count;
 538
 539         if (scsi_sg_count(sc)) {
 540                 bd_count = qedf_map_sg(io_req);
 541                 if (bd_count == 0)
 542                         return -ENOMEM;
 543         } else {
 544                 bd_count = 0;
 545                 bd[0].sge_addr.lo = bd[0].sge_addr.hi = 0;
 546                 bd[0].sge_len = 0;
 547         }
 548         io_req->bd_tbl->bd_valid = bd_count;
 549
 550         return 0;
 551 }
 552
 553 static void qedf_build_fcp_cmnd(struct qedf_ioreq *io_req,
 554                                   struct fcp_cmnd *fcp_cmnd)
 555 {
 556         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 557
 558         /* fcp_cmnd is 32 bytes */
 559         memset(fcp_cmnd, 0, FCP_CMND_LEN);
 560
 561         /* 8 bytes: SCSI LUN info */
 562         int_to_scsilun(sc_cmd->device->lun,
 563                         (struct scsi_lun *)&fcp_cmnd->fc_lun);
 564
 565         /* 4 bytes: flag info */
 566         fcp_cmnd->fc_pri_ta = 0;
 567         fcp_cmnd->fc_tm_flags = io_req->tm_flags;
 568         fcp_cmnd->fc_flags = io_req->io_req_flags;
 569         fcp_cmnd->fc_cmdref = 0;
 570
 571         /* Populate data direction */
 572         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
 573                 fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
 574         } else {
 575                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE)
 576                         fcp_cmnd->fc_flags |= FCP_CFL_WRDATA;
 577                 else if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE)
 578                         fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
 579         }
 580
 581         fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
 582
 583         /* 16 bytes: CDB information */
 584         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD)
 585                 memcpy(fcp_cmnd->fc_cdb, sc_cmd->cmnd, sc_cmd->cmd_len);
 586
 587         /* 4 bytes: FCP data length */
 588         fcp_cmnd->fc_dl = htonl(io_req->data_xfer_len);
 589 }
 590
 591 static void  qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
 592         struct qedf_ioreq *io_req, struct e4_fcoe_task_context *task_ctx,
 593         struct fcoe_wqe *sqe)
 594 {
 595         enum fcoe_task_type task_type;
 596         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 597         struct io_bdt *bd_tbl = io_req->bd_tbl;
 598         u8 fcp_cmnd[32];
 599         u32 tmp_fcp_cmnd[8];
 600         int bd_count = 0;
 601         struct qedf_ctx *qedf = fcport->qedf;
 602         uint16_t cq_idx = smp_processor_id() % qedf->num_queues;
 603         struct regpair sense_data_buffer_phys_addr;
 604         u32 tx_io_size = 0;
 605         u32 rx_io_size = 0;
 606         int i, cnt;
 607
 608         /* Note init_initiator_rw_fcoe_task memsets the task context */
 609         io_req->task = task_ctx;
 610         memset(task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 611         memset(io_req->task_params, 0, sizeof(struct fcoe_task_params));
 612         memset(io_req->sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 613
 614         /* Set task type bassed on DMA directio of command */
 615         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
 616                 task_type = FCOE_TASK_TYPE_READ_INITIATOR;
 617         } else {
 618                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
 619                         task_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
 620                         tx_io_size = io_req->data_xfer_len;
 621                 } else {
 622                         task_type = FCOE_TASK_TYPE_READ_INITIATOR;
 623                         rx_io_size = io_req->data_xfer_len;
 624                 }
 625         }
 626
 627         /* Setup the fields for fcoe_task_params */
 628         io_req->task_params->context = task_ctx;
 629         io_req->task_params->sqe = sqe;
 630         io_req->task_params->task_type = task_type;
 631         io_req->task_params->tx_io_size = tx_io_size;
 632         io_req->task_params->rx_io_size = rx_io_size;
 633         io_req->task_params->conn_cid = fcport->fw_cid;
 634         io_req->task_params->itid = io_req->xid;
 635         io_req->task_params->cq_rss_number = cq_idx;
 636         io_req->task_params->is_tape_device = fcport->dev_type;
 637
 638         /* Fill in information for scatter/gather list */
 639         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD) {
 640                 bd_count = bd_tbl->bd_valid;
 641                 io_req->sgl_task_params->sgl = bd_tbl->bd_tbl;
 642                 io_req->sgl_task_params->sgl_phys_addr.lo =
 643                         U64_LO(bd_tbl->bd_tbl_dma);
 644                 io_req->sgl_task_params->sgl_phys_addr.hi =
 645                         U64_HI(bd_tbl->bd_tbl_dma);
 646                 io_req->sgl_task_params->num_sges = bd_count;
 647                 io_req->sgl_task_params->total_buffer_size =
 648                     scsi_bufflen(io_req->sc_cmd);
 649                 if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 650                         io_req->sgl_task_params->small_mid_sge = 1;
 651                 else
 652                         io_req->sgl_task_params->small_mid_sge = 0;
 653         }
 654
 655         /* Fill in physical address of sense buffer */
 656         sense_data_buffer_phys_addr.lo = U64_LO(io_req->sense_buffer_dma);
 657         sense_data_buffer_phys_addr.hi = U64_HI(io_req->sense_buffer_dma);
 658
 659         /* fill FCP_CMND IU */
 660         qedf_build_fcp_cmnd(io_req, (struct fcp_cmnd *)tmp_fcp_cmnd);
 661
 662         /* Swap fcp_cmnd since FC is big endian */
 663         cnt = sizeof(struct fcp_cmnd) / sizeof(u32);
 664         for (i = 0; i < cnt; i++) {
 665                 tmp_fcp_cmnd[i] = cpu_to_be32(tmp_fcp_cmnd[i]);
 666         }
 667         memcpy(fcp_cmnd, tmp_fcp_cmnd, sizeof(struct fcp_cmnd));
 668
 669         init_initiator_rw_fcoe_task(io_req->task_params,
 670                                     io_req->sgl_task_params,
 671                                     sense_data_buffer_phys_addr,
 672                                     io_req->task_retry_identifier, fcp_cmnd);
 673
 674         /* Increment SGL type counters */
 675         if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 676                 qedf->slow_sge_ios++;
 677         else
 678                 qedf->fast_sge_ios++;
 679 }
 680
 681 void qedf_init_mp_task(struct qedf_ioreq *io_req,
 682         struct e4_fcoe_task_context *task_ctx, struct fcoe_wqe *sqe)
 683 {
 684         struct qedf_mp_req *mp_req = &(io_req->mp_req);
 685         struct qedf_rport *fcport = io_req->fcport;
 686         struct qedf_ctx *qedf = io_req->fcport->qedf;
 687         struct fc_frame_header *fc_hdr;
 688         struct fcoe_tx_mid_path_params task_fc_hdr;
 689         struct scsi_sgl_task_params tx_sgl_task_params;
 690         struct scsi_sgl_task_params rx_sgl_task_params;
 691
 692         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 693                   "Initializing MP task for cmd_type=%d\n",
 694                   io_req->cmd_type);
 695
 696         qedf->control_requests++;
 697
 698         memset(&tx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 699         memset(&rx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 700         memset(task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 701         memset(&task_fc_hdr, 0, sizeof(struct fcoe_tx_mid_path_params));
 702
 703         /* Setup the task from io_req for easy reference */
 704         io_req->task = task_ctx;
 705
 706         /* Setup the fields for fcoe_task_params */
 707         io_req->task_params->context = task_ctx;
 708         io_req->task_params->sqe = sqe;
 709         io_req->task_params->task_type = FCOE_TASK_TYPE_MIDPATH;
 710         io_req->task_params->tx_io_size = io_req->data_xfer_len;
 711         /* rx_io_size tells the f/w how large a response buffer we have */
 712         io_req->task_params->rx_io_size = PAGE_SIZE;
 713         io_req->task_params->conn_cid = fcport->fw_cid;
 714         io_req->task_params->itid = io_req->xid;
 715         /* Return middle path commands on CQ 0 */
 716         io_req->task_params->cq_rss_number = 0;
 717         io_req->task_params->is_tape_device = fcport->dev_type;
 718
 719         fc_hdr = &(mp_req->req_fc_hdr);
 720         /* Set OX_ID and RX_ID based on driver task id */
 721         fc_hdr->fh_ox_id = io_req->xid;
 722         fc_hdr->fh_rx_id = htons(0xffff);
 723
 724         /* Set up FC header information */
 725         task_fc_hdr.parameter = fc_hdr->fh_parm_offset;
 726         task_fc_hdr.r_ctl = fc_hdr->fh_r_ctl;
 727         task_fc_hdr.type = fc_hdr->fh_type;
 728         task_fc_hdr.cs_ctl = fc_hdr->fh_cs_ctl;
 729         task_fc_hdr.df_ctl = fc_hdr->fh_df_ctl;
 730         task_fc_hdr.rx_id = fc_hdr->fh_rx_id;
 731         task_fc_hdr.ox_id = fc_hdr->fh_ox_id;
 732
 733         /* Set up s/g list parameters for request buffer */
 734         tx_sgl_task_params.sgl = mp_req->mp_req_bd;
 735         tx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_req_bd_dma);
 736         tx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_req_bd_dma);
 737         tx_sgl_task_params.num_sges = 1;
 738         /* Set PAGE_SIZE for now since sg element is that size ??? */
 739         tx_sgl_task_params.total_buffer_size = io_req->data_xfer_len;
 740         tx_sgl_task_params.small_mid_sge = 0;
 741
 742         /* Set up s/g list parameters for request buffer */
 743         rx_sgl_task_params.sgl = mp_req->mp_resp_bd;
 744         rx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_resp_bd_dma);
 745         rx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_resp_bd_dma);
 746         rx_sgl_task_params.num_sges = 1;
 747         /* Set PAGE_SIZE for now since sg element is that size ??? */
 748         rx_sgl_task_params.total_buffer_size = PAGE_SIZE;
 749         rx_sgl_task_params.small_mid_sge = 0;
 750
 751
 752         /*
 753          * Last arg is 0 as previous code did not set that we wanted the
 754          * fc header information.
 755          */
 756         init_initiator_midpath_unsolicited_fcoe_task(io_req->task_params,
 757                                                      &task_fc_hdr,
 758                                                      &tx_sgl_task_params,
 759                                                      &rx_sgl_task_params, 0);
 760 }
 761
 762 /* Presumed that fcport->rport_lock is held */
 763 u16 qedf_get_sqe_idx(struct qedf_rport *fcport)
 764 {
 765         uint16_t total_sqe = (fcport->sq_mem_size)/(sizeof(struct fcoe_wqe));
 766         u16 rval;
 767
 768         rval = fcport->sq_prod_idx;
 769
 770         /* Adjust ring index */
 771         fcport->sq_prod_idx++;
 772         fcport->fw_sq_prod_idx++;
 773         if (fcport->sq_prod_idx == total_sqe)
 774                 fcport->sq_prod_idx = 0;
 775
 776         return rval;
 777 }
 778
 779 void qedf_ring_doorbell(struct qedf_rport *fcport)
 780 {
 781         struct fcoe_db_data dbell = { 0 };
 782
 783         dbell.agg_flags = 0;
 784
 785         dbell.params |= DB_DEST_XCM << FCOE_DB_DATA_DEST_SHIFT;
 786         dbell.params |= DB_AGG_CMD_SET << FCOE_DB_DATA_AGG_CMD_SHIFT;
 787         dbell.params |= DQ_XCM_FCOE_SQ_PROD_CMD <<
 788             FCOE_DB_DATA_AGG_VAL_SEL_SHIFT;
 789
 790         dbell.sq_prod = fcport->fw_sq_prod_idx;
 791         /* wmb makes sure that the BDs data is updated before updating the
 792          * producer, otherwise FW may read old data from the BDs.
 793          */
 794         wmb();
 795         barrier();
 796         writel(*(u32 *)&dbell, fcport->p_doorbell);
 797         /*
 798          * Fence required to flush the write combined buffer, since another
 799          * CPU may write to the same doorbell address and data may be lost
 800          * due to relaxed order nature of write combined bar.
 801          */
 802         wmb();
 803 }
 804
 805 static void qedf_trace_io(struct qedf_rport *fcport, struct qedf_ioreq *io_req,
 806                           int8_t direction)
 807 {
 808         struct qedf_ctx *qedf = fcport->qedf;
 809         struct qedf_io_log *io_log;
 810         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 811         unsigned long flags;
 812         uint8_t op;
 813
 814         spin_lock_irqsave(&qedf->io_trace_lock, flags);
 815
 816         io_log = &qedf->io_trace_buf[qedf->io_trace_idx];
 817         io_log->direction = direction;
 818         io_log->task_id = io_req->xid;
 819         io_log->port_id = fcport->rdata->ids.port_id;
 820         io_log->lun = sc_cmd->device->lun;
 821         io_log->op = op = sc_cmd->cmnd[0];
 822         io_log->lba[0] = sc_cmd->cmnd[2];
 823         io_log->lba[1] = sc_cmd->cmnd[3];
 824         io_log->lba[2] = sc_cmd->cmnd[4];
 825         io_log->lba[3] = sc_cmd->cmnd[5];
 826         io_log->bufflen = scsi_bufflen(sc_cmd);
 827         io_log->sg_count = scsi_sg_count(sc_cmd);
 828         io_log->result = sc_cmd->result;
 829         io_log->jiffies = jiffies;
 830         io_log->refcount = kref_read(&io_req->refcount);
 831
 832         if (direction == QEDF_IO_TRACE_REQ) {
 833                 /* For requests we only care abot the submission CPU */
 834                 io_log->req_cpu = io_req->cpu;
 835                 io_log->int_cpu = 0;
 836                 io_log->rsp_cpu = 0;
 837         } else if (direction == QEDF_IO_TRACE_RSP) {
 838                 io_log->req_cpu = io_req->cpu;
 839                 io_log->int_cpu = io_req->int_cpu;
 840                 io_log->rsp_cpu = smp_processor_id();
 841         }
 842
 843         io_log->sge_type = io_req->sge_type;
 844
 845         qedf->io_trace_idx++;
 846         if (qedf->io_trace_idx == QEDF_IO_TRACE_SIZE)
 847                 qedf->io_trace_idx = 0;
 848
 849         spin_unlock_irqrestore(&qedf->io_trace_lock, flags);
 850 }
 851
 852 int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 853 {
 854         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 855         struct Scsi_Host *host = sc_cmd->device->host;
 856         struct fc_lport *lport = shost_priv(host);
 857         struct qedf_ctx *qedf = lport_priv(lport);
 858         struct e4_fcoe_task_context *task_ctx;
 859         u16 xid;
 860         struct fcoe_wqe *sqe;
 861         u16 sqe_idx;
 862
 863         /* Initialize rest of io_req fileds */
 864         io_req->data_xfer_len = scsi_bufflen(sc_cmd);
 865         sc_cmd->SCp.ptr = (char *)io_req;
 866         io_req->sge_type = QEDF_IOREQ_FAST_SGE; /* Assume fast SGL by default */
 867
 868         /* Record which cpu this request is associated with */
 869         io_req->cpu = smp_processor_id();
 870
 871         if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
 872                 io_req->io_req_flags = QEDF_READ;
 873                 qedf->input_requests++;
 874         } else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
 875                 io_req->io_req_flags = QEDF_WRITE;
 876                 qedf->output_requests++;
 877         } else {
 878                 io_req->io_req_flags = 0;
 879                 qedf->control_requests++;
 880         }
 881
 882         xid = io_req->xid;
 883
 884         /* Build buffer descriptor list for firmware from sg list */
 885         if (qedf_build_bd_list_from_sg(io_req)) {
 886                 QEDF_ERR(&(qedf->dbg_ctx), "BD list creation failed.\n");
 887                 /* Release cmd will release io_req, but sc_cmd is assigned */
 888                 io_req->sc_cmd = NULL;
 889                 kref_put(&io_req->refcount, qedf_release_cmd);
 890                 return -EAGAIN;
 891         }
 892
 893         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
 894             test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 895                 QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
 896                 /* Release cmd will release io_req, but sc_cmd is assigned */
 897                 io_req->sc_cmd = NULL;
 898                 kref_put(&io_req->refcount, qedf_release_cmd);
 899                 return -EINVAL;
 900         }
 901
 902         /* Record LUN number for later use if we neeed them */
 903         io_req->lun = (int)sc_cmd->device->lun;
 904
 905         /* Obtain free SQE */
 906         sqe_idx = qedf_get_sqe_idx(fcport);
 907         sqe = &fcport->sq[sqe_idx];
 908         memset(sqe, 0, sizeof(struct fcoe_wqe));
 909
 910         /* Get the task context */
 911         task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
 912         if (!task_ctx) {
 913                 QEDF_WARN(&(qedf->dbg_ctx), "task_ctx is NULL, xid=%d.\n",
 914                            xid);
 915                 /* Release cmd will release io_req, but sc_cmd is assigned */
 916                 io_req->sc_cmd = NULL;
 917                 kref_put(&io_req->refcount, qedf_release_cmd);
 918                 return -EINVAL;
 919         }
 920
 921         qedf_init_task(fcport, lport, io_req, task_ctx, sqe);
 922
 923         /* Ring doorbell */
 924         qedf_ring_doorbell(fcport);
 925
 926         /* Set that command is with the firmware now */
 927         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
 928
 929         if (qedf_io_tracing && io_req->sc_cmd)
 930                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_REQ);
 931
 932         return false;
 933 }
 934
 935 int
 936 qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 937 {
 938         struct fc_lport *lport = shost_priv(host);
 939         struct qedf_ctx *qedf = lport_priv(lport);
 940         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
 941         struct fc_rport_libfc_priv *rp = rport->dd_data;
 942         struct qedf_rport *fcport;
 943         struct qedf_ioreq *io_req;
 944         int rc = 0;
 945         int rval;
 946         unsigned long flags = 0;
 947         int num_sgs = 0;
 948
 949         num_sgs = scsi_sg_count(sc_cmd);
 950         if (scsi_sg_count(sc_cmd) > QEDF_MAX_BDS_PER_CMD) {
 951                 QEDF_ERR(&qedf->dbg_ctx,
 952                          "Number of SG elements %d exceeds what hardware limitation of %d.\n",
 953                          num_sgs, QEDF_MAX_BDS_PER_CMD);
 954                 sc_cmd->result = DID_ERROR;
 955                 sc_cmd->scsi_done(sc_cmd);
 956                 return 0;
 957         }
 958
 959         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
 960             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
 961                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 962                           "Returning DNC as unloading or stop io, flags 0x%lx.\n",
 963                           qedf->flags);
 964                 sc_cmd->result = DID_NO_CONNECT << 16;
 965                 sc_cmd->scsi_done(sc_cmd);
 966                 return 0;
 967         }
 968
 969         if (!qedf->pdev->msix_enabled) {
 970                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 971                     "Completing sc_cmd=%p DID_NO_CONNECT as MSI-X is not enabled.\n",
 972                     sc_cmd);
 973                 sc_cmd->result = DID_NO_CONNECT << 16;
 974                 sc_cmd->scsi_done(sc_cmd);
 975                 return 0;
 976         }
 977
 978         rval = fc_remote_port_chkready(rport);
 979         if (rval) {
 980                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 981                           "fc_remote_port_chkready failed=0x%x for port_id=0x%06x.\n",
 982                           rval, rport->port_id);
 983                 sc_cmd->result = rval;
 984                 sc_cmd->scsi_done(sc_cmd);
 985                 return 0;
 986         }
 987
 988         /* Retry command if we are doing a qed drain operation */
 989         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
 990                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Drain active.\n");
 991                 rc = SCSI_MLQUEUE_HOST_BUSY;
 992                 goto exit_qcmd;
 993         }
 994
 995         if (lport->state != LPORT_ST_READY ||
 996             atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
 997                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Link down.\n");
 998                 rc = SCSI_MLQUEUE_HOST_BUSY;
 999                 goto exit_qcmd;
1000         }
1001
1002         /* rport and tgt are allocated together, so tgt should be non-NULL */
1003         fcport = (struct qedf_rport *)&rp[1];
1004
1005         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
1006             test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
1007                 /*
1008                  * Session is not offloaded yet. Let SCSI-ml retry
1009                  * the command.
1010                  */
1011                 rc = SCSI_MLQUEUE_TARGET_BUSY;
1012                 goto exit_qcmd;
1013         }
1014
1015         atomic_inc(&fcport->ios_to_queue);
1016
1017         if (fcport->retry_delay_timestamp) {
1018                 /* Take fcport->rport_lock for resetting the delay_timestamp */
1019                 spin_lock_irqsave(&fcport->rport_lock, flags);
1020                 if (time_after(jiffies, fcport->retry_delay_timestamp)) {
1021                         fcport->retry_delay_timestamp = 0;
1022                 } else {
1023                         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1024                         /* If retry_delay timer is active, flow off the ML */
1025                         rc = SCSI_MLQUEUE_TARGET_BUSY;
1026                         atomic_dec(&fcport->ios_to_queue);
1027                         goto exit_qcmd;
1028                 }
1029                 spin_unlock_irqrestore(&fcport->rport_lock, flags);
1030         }
1031
1032         io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
1033         if (!io_req) {
1034                 rc = SCSI_MLQUEUE_HOST_BUSY;
1035                 atomic_dec(&fcport->ios_to_queue);
1036                 goto exit_qcmd;
1037         }
1038
1039         io_req->sc_cmd = sc_cmd;
1040
1041         /* Take fcport->rport_lock for posting to fcport send queue */
1042         spin_lock_irqsave(&fcport->rport_lock, flags);
1043         if (qedf_post_io_req(fcport, io_req)) {
1044                 QEDF_WARN(&(qedf->dbg_ctx), "Unable to post io_req\n");
1045                 /* Return SQE to pool */
1046                 atomic_inc(&fcport->free_sqes);
1047                 rc = SCSI_MLQUEUE_HOST_BUSY;
1048         }
1049         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1050         atomic_dec(&fcport->ios_to_queue);
1051
1052 exit_qcmd:
1053         return rc;
1054 }
1055
1056 static void qedf_parse_fcp_rsp(struct qedf_ioreq *io_req,
1057                                  struct fcoe_cqe_rsp_info *fcp_rsp)
1058 {
1059         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
1060         struct qedf_ctx *qedf = io_req->fcport->qedf;
1061         u8 rsp_flags = fcp_rsp->rsp_flags.flags;
1062         int fcp_sns_len = 0;
1063         int fcp_rsp_len = 0;
1064         uint8_t *rsp_info, *sense_data;
1065
1066         io_req->fcp_status = FC_GOOD;
1067         io_req->fcp_resid = 0;
1068         if (rsp_flags & (FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER |
1069             FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER))
1070                 io_req->fcp_resid = fcp_rsp->fcp_resid;
1071
1072         io_req->scsi_comp_flags = rsp_flags;
1073         CMD_SCSI_STATUS(sc_cmd) = io_req->cdb_status =
1074             fcp_rsp->scsi_status_code;
1075
1076         if (rsp_flags &
1077             FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID)
1078                 fcp_rsp_len = fcp_rsp->fcp_rsp_len;
1079
1080         if (rsp_flags &
1081             FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID)
1082                 fcp_sns_len = fcp_rsp->fcp_sns_len;
1083
1084         io_req->fcp_rsp_len = fcp_rsp_len;
1085         io_req->fcp_sns_len = fcp_sns_len;
1086         rsp_info = sense_data = io_req->sense_buffer;
1087
1088         /* fetch fcp_rsp_code */
1089         if ((fcp_rsp_len == 4) || (fcp_rsp_len == 8)) {
1090                 /* Only for task management function */
1091                 io_req->fcp_rsp_code = rsp_info[3];
1092                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1093                     "fcp_rsp_code = %d\n", io_req->fcp_rsp_code);
1094                 /* Adjust sense-data location. */
1095                 sense_data += fcp_rsp_len;
1096         }
1097
1098         if (fcp_sns_len > SCSI_SENSE_BUFFERSIZE) {
1099                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1100                     "Truncating sense buffer\n");
1101                 fcp_sns_len = SCSI_SENSE_BUFFERSIZE;
1102         }
1103
1104         /* The sense buffer can be NULL for TMF commands */
1105         if (sc_cmd->sense_buffer) {
1106                 memset(sc_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
1107                 if (fcp_sns_len)
1108                         memcpy(sc_cmd->sense_buffer, sense_data,
1109                             fcp_sns_len);
1110         }
1111 }
1112
1113 static void qedf_unmap_sg_list(struct qedf_ctx *qedf, struct qedf_ioreq *io_req)
1114 {
1115         struct scsi_cmnd *sc = io_req->sc_cmd;
1116
1117         if (io_req->bd_tbl->bd_valid && sc && scsi_sg_count(sc)) {
1118                 dma_unmap_sg(&qedf->pdev->dev, scsi_sglist(sc),
1119                     scsi_sg_count(sc), sc->sc_data_direction);
1120                 io_req->bd_tbl->bd_valid = 0;
1121         }
1122 }
1123
1124 void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1125         struct qedf_ioreq *io_req)
1126 {
1127         struct scsi_cmnd *sc_cmd;
1128         struct fcoe_cqe_rsp_info *fcp_rsp;
1129         struct qedf_rport *fcport;
1130         int refcount;
1131         u16 scope, qualifier = 0;
1132         u8 fw_residual_flag = 0;
1133         unsigned long flags = 0;
1134         u16 chk_scope = 0;
1135
1136         if (!io_req)
1137                 return;
1138         if (!cqe)
1139                 return;
1140
1141         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1142             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
1143             test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
1144                 QEDF_ERR(&qedf->dbg_ctx,
1145                          "io_req xid=0x%x already in cleanup or abort processing or already completed.\n",
1146                          io_req->xid);
1147                 return;
1148         }
1149
1150         sc_cmd = io_req->sc_cmd;
1151         fcp_rsp = &cqe->cqe_info.rsp_info;
1152
1153         if (!sc_cmd) {
1154                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1155                 return;
1156         }
1157
1158         if (!sc_cmd->SCp.ptr) {
1159                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1160                     "another context.\n");
1161                 return;
1162         }
1163
1164         if (!sc_cmd->device) {
1165                 QEDF_ERR(&qedf->dbg_ctx,
1166                          "Device for sc_cmd %p is NULL.\n", sc_cmd);
1167                 return;
1168         }
1169
1170         if (!sc_cmd->request) {
1171                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd->request is NULL, "
1172                     "sc_cmd=%p.\n", sc_cmd);
1173                 return;
1174         }
1175
1176         if (!sc_cmd->request->q) {
1177                 QEDF_WARN(&(qedf->dbg_ctx), "request->q is NULL so request "
1178                    "is not valid, sc_cmd=%p.\n", sc_cmd);
1179                 return;
1180         }
1181
1182         fcport = io_req->fcport;
1183
1184         /*
1185          * When flush is active, let the cmds be completed from the cleanup
1186          * context
1187          */
1188         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
1189             (test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags) &&
1190              sc_cmd->device->lun == (u64)fcport->lun_reset_lun)) {
1191                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1192                           "Dropping good completion xid=0x%x as fcport is flushing",
1193                           io_req->xid);
1194                 return;
1195         }
1196
1197         qedf_parse_fcp_rsp(io_req, fcp_rsp);
1198
1199         qedf_unmap_sg_list(qedf, io_req);
1200
1201         /* Check for FCP transport error */
1202         if (io_req->fcp_rsp_len > 3 && io_req->fcp_rsp_code) {
1203                 QEDF_ERR(&(qedf->dbg_ctx),
1204                     "FCP I/O protocol failure xid=0x%x fcp_rsp_len=%d "
1205                     "fcp_rsp_code=%d.\n", io_req->xid, io_req->fcp_rsp_len,
1206                     io_req->fcp_rsp_code);
1207                 sc_cmd->result = DID_BUS_BUSY << 16;
1208                 goto out;
1209         }
1210
1211         fw_residual_flag = GET_FIELD(cqe->cqe_info.rsp_info.fw_error_flags,
1212             FCOE_CQE_RSP_INFO_FW_UNDERRUN);
1213         if (fw_residual_flag) {
1214                 QEDF_ERR(&qedf->dbg_ctx,
1215                          "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x fcp_resid=%d fw_residual=0x%x lba=%02x%02x%02x%02x.\n",
1216                          io_req->xid, fcp_rsp->rsp_flags.flags,
1217                          io_req->fcp_resid,
1218                          cqe->cqe_info.rsp_info.fw_residual, sc_cmd->cmnd[2],
1219                          sc_cmd->cmnd[3], sc_cmd->cmnd[4], sc_cmd->cmnd[5]);
1220
1221                 if (io_req->cdb_status == 0)
1222                         sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
1223                 else
1224                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1225
1226                 /*
1227                  * Set resid to the whole buffer length so we won't try to resue
1228                  * any previously data.
1229                  */
1230                 scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1231                 goto out;
1232         }
1233
1234         switch (io_req->fcp_status) {
1235         case FC_GOOD:
1236                 if (io_req->cdb_status == 0) {
1237                         /* Good I/O completion */
1238                         sc_cmd->result = DID_OK << 16;
1239                 } else {
1240                         refcount = kref_read(&io_req->refcount);
1241                         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1242                             "%d:0:%d:%lld xid=0x%0x op=0x%02x "
1243                             "lba=%02x%02x%02x%02x cdb_status=%d "
1244                             "fcp_resid=0x%x refcount=%d.\n",
1245                             qedf->lport->host->host_no, sc_cmd->device->id,
1246                             sc_cmd->device->lun, io_req->xid,
1247                             sc_cmd->cmnd[0], sc_cmd->cmnd[2], sc_cmd->cmnd[3],
1248                             sc_cmd->cmnd[4], sc_cmd->cmnd[5],
1249                             io_req->cdb_status, io_req->fcp_resid,
1250                             refcount);
1251                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1252
1253                         if (io_req->cdb_status == SAM_STAT_TASK_SET_FULL ||
1254                             io_req->cdb_status == SAM_STAT_BUSY) {
1255                                 /*
1256                                  * Check whether we need to set retry_delay at
1257                                  * all based on retry_delay module parameter
1258                                  * and the status qualifier.
1259                                  */
1260
1261                                 /* Upper 2 bits */
1262                                 scope = fcp_rsp->retry_delay_timer & 0xC000;
1263                                 /* Lower 14 bits */
1264                                 qualifier = fcp_rsp->retry_delay_timer & 0x3FFF;
1265
1266                                 if (qedf_retry_delay)
1267                                         chk_scope = 1;
1268                                 /* Record stats */
1269                                 if (io_req->cdb_status ==
1270                                     SAM_STAT_TASK_SET_FULL)
1271                                         qedf->task_set_fulls++;
1272                                 else
1273                                         qedf->busy++;
1274                         }
1275                 }
1276                 if (io_req->fcp_resid)
1277                         scsi_set_resid(sc_cmd, io_req->fcp_resid);
1278
1279                 if (chk_scope == 1) {
1280                         if ((scope == 1 || scope == 2) &&
1281                             (qualifier > 0 && qualifier <= 0x3FEF)) {
1282                                 /* Check we don't go over the max */
1283                                 if (qualifier > QEDF_RETRY_DELAY_MAX) {
1284                                         qualifier = QEDF_RETRY_DELAY_MAX;
1285                                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1286                                                   "qualifier = %d\n",
1287                                                   (fcp_rsp->retry_delay_timer &
1288                                                   0x3FFF));
1289                                 }
1290                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1291                                           "Scope = %d and qualifier = %d",
1292                                           scope, qualifier);
1293                                 /*  Take fcport->rport_lock to
1294                                  *  update the retry_delay_timestamp
1295                                  */
1296                                 spin_lock_irqsave(&fcport->rport_lock, flags);
1297                                 fcport->retry_delay_timestamp =
1298                                         jiffies + (qualifier * HZ / 10);
1299                                 spin_unlock_irqrestore(&fcport->rport_lock,
1300                                                        flags);
1301
1302                         } else {
1303                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1304                                           "combination of scope = %d and qualifier = %d is not handled in qedf.\n",
1305                                           scope, qualifier);
1306                         }
1307                 }
1308                 break;
1309         default:
1310                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "fcp_status=%d.\n",
1311                            io_req->fcp_status);
1312                 break;
1313         }
1314
1315 out:
1316         if (qedf_io_tracing)
1317                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_RSP);
1318
1319         /*
1320          * We wait till the end of the function to clear the
1321          * outstanding bit in case we need to send an abort
1322          */
1323         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
1324
1325         io_req->sc_cmd = NULL;
1326         sc_cmd->SCp.ptr =  NULL;
1327         sc_cmd->scsi_done(sc_cmd);
1328         kref_put(&io_req->refcount, qedf_release_cmd);
1329 }
1330
1331 /* Return a SCSI command in some other context besides a normal completion */
1332 void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
1333         int result)
1334 {
1335         struct scsi_cmnd *sc_cmd;
1336         int refcount;
1337
1338         if (!io_req) {
1339                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "io_req is NULL\n");
1340                 return;
1341         }
1342
1343         if (test_and_set_bit(QEDF_CMD_ERR_SCSI_DONE, &io_req->flags)) {
1344                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1345                           "io_req:%p scsi_done handling already done\n",
1346                           io_req);
1347                 return;
1348         }
1349
1350         /*
1351          * We will be done with this command after this call so clear the
1352          * outstanding bit.
1353          */
1354         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
1355
1356         sc_cmd = io_req->sc_cmd;
1357
1358         if (!sc_cmd) {
1359                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1360                 return;
1361         }
1362
1363         if (!virt_addr_valid(sc_cmd)) {
1364                 QEDF_ERR(&qedf->dbg_ctx, "sc_cmd=%p is not valid.", sc_cmd);
1365                 goto bad_scsi_ptr;
1366         }
1367
1368         if (!sc_cmd->SCp.ptr) {
1369                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1370                     "another context.\n");
1371                 return;
1372         }
1373
1374         if (!sc_cmd->device) {
1375                 QEDF_ERR(&qedf->dbg_ctx, "Device for sc_cmd %p is NULL.\n",
1376                          sc_cmd);
1377                 goto bad_scsi_ptr;
1378         }
1379
1380         if (!virt_addr_valid(sc_cmd->device)) {
1381                 QEDF_ERR(&qedf->dbg_ctx,
1382                          "Device pointer for sc_cmd %p is bad.\n", sc_cmd);
1383                 goto bad_scsi_ptr;
1384         }
1385
1386         if (!sc_cmd->sense_buffer) {
1387                 QEDF_ERR(&qedf->dbg_ctx,
1388                          "sc_cmd->sense_buffer for sc_cmd %p is NULL.\n",
1389                          sc_cmd);
1390                 goto bad_scsi_ptr;
1391         }
1392
1393         if (!virt_addr_valid(sc_cmd->sense_buffer)) {
1394                 QEDF_ERR(&qedf->dbg_ctx,
1395                          "sc_cmd->sense_buffer for sc_cmd %p is bad.\n",
1396                          sc_cmd);
1397                 goto bad_scsi_ptr;
1398         }
1399
1400         if (!sc_cmd->scsi_done) {
1401                 QEDF_ERR(&qedf->dbg_ctx,
1402                          "sc_cmd->scsi_done for sc_cmd %p is NULL.\n",
1403                          sc_cmd);
1404                 goto bad_scsi_ptr;
1405         }
1406
1407         qedf_unmap_sg_list(qedf, io_req);
1408
1409         sc_cmd->result = result << 16;
1410         refcount = kref_read(&io_req->refcount);
1411         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "%d:0:%d:%lld: Completing "
1412             "sc_cmd=%p result=0x%08x op=0x%02x lba=0x%02x%02x%02x%02x, "
1413             "allowed=%d retries=%d refcount=%d.\n",
1414             qedf->lport->host->host_no, sc_cmd->device->id,
1415             sc_cmd->device->lun, sc_cmd, sc_cmd->result, sc_cmd->cmnd[0],
1416             sc_cmd->cmnd[2], sc_cmd->cmnd[3], sc_cmd->cmnd[4],
1417             sc_cmd->cmnd[5], sc_cmd->allowed, sc_cmd->retries,
1418             refcount);
1419
1420         /*
1421          * Set resid to the whole buffer length so we won't try to resue any
1422          * previously read data
1423          */
1424         scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1425
1426         if (qedf_io_tracing)
1427                 qedf_trace_io(io_req->fcport, io_req, QEDF_IO_TRACE_RSP);
1428
1429         io_req->sc_cmd = NULL;
1430         sc_cmd->SCp.ptr = NULL;
1431         sc_cmd->scsi_done(sc_cmd);
1432         kref_put(&io_req->refcount, qedf_release_cmd);
1433         return;
1434
1435 bad_scsi_ptr:
1436         /*
1437          * Clear the io_req->sc_cmd backpointer so we don't try to process
1438          * this again
1439          */
1440         io_req->sc_cmd = NULL;
1441         kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 001 */
1442 }
1443
1444 /*
1445  * Handle warning type CQE completions. This is mainly used for REC timer
1446  * popping.
1447  */
1448 void qedf_process_warning_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1449         struct qedf_ioreq *io_req)
1450 {
1451         int rval, i;
1452         struct qedf_rport *fcport = io_req->fcport;
1453         u64 err_warn_bit_map;
1454         u8 err_warn = 0xff;
1455
1456         if (!cqe) {
1457                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1458                           "cqe is NULL for io_req %p xid=0x%x\n",
1459                           io_req, io_req->xid);
1460                 return;
1461         }
1462
1463         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Warning CQE, "
1464                   "xid=0x%x\n", io_req->xid);
1465         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1466                   "err_warn_bitmap=%08x:%08x\n",
1467                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1468                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1469         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1470                   "rx_buff_off=%08x, rx_id=%04x\n",
1471                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1472                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1473                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1474
1475         /* Normalize the error bitmap value to an just an unsigned int */
1476         err_warn_bit_map = (u64)
1477             ((u64)cqe->cqe_info.err_info.err_warn_bitmap_hi << 32) |
1478             (u64)cqe->cqe_info.err_info.err_warn_bitmap_lo;
1479         for (i = 0; i < 64; i++) {
1480                 if (err_warn_bit_map & (u64)((u64)1 << i)) {
1481                         err_warn = i;
1482                         break;
1483                 }
1484         }
1485
1486         /* Check if REC TOV expired if this is a tape device */
1487         if (fcport->dev_type == QEDF_RPORT_TYPE_TAPE) {
1488                 if (err_warn ==
1489                     FCOE_WARNING_CODE_REC_TOV_TIMER_EXPIRATION) {
1490                         QEDF_ERR(&(qedf->dbg_ctx), "REC timer expired.\n");
1491                         if (!test_bit(QEDF_CMD_SRR_SENT, &io_req->flags)) {
1492                                 io_req->rx_buf_off =
1493                                     cqe->cqe_info.err_info.rx_buf_off;
1494                                 io_req->tx_buf_off =
1495                                     cqe->cqe_info.err_info.tx_buf_off;
1496                                 io_req->rx_id = cqe->cqe_info.err_info.rx_id;
1497                                 rval = qedf_send_rec(io_req);
1498                                 /*
1499                                  * We only want to abort the io_req if we
1500                                  * can't queue the REC command as we want to
1501                                  * keep the exchange open for recovery.
1502                                  */
1503                                 if (rval)
1504                                         goto send_abort;
1505                         }
1506                         return;
1507                 }
1508         }
1509
1510 send_abort:
1511         init_completion(&io_req->abts_done);
1512         rval = qedf_initiate_abts(io_req, true);
1513         if (rval)
1514                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1515 }
1516
1517 /* Cleanup a command when we receive an error detection completion */
1518 void qedf_process_error_detect(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1519         struct qedf_ioreq *io_req)
1520 {
1521         int rval;
1522
1523         if (!cqe) {
1524                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1525                           "cqe is NULL for io_req %p\n", io_req);
1526                 return;
1527         }
1528
1529         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Error detection CQE, "
1530                   "xid=0x%x\n", io_req->xid);
1531         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1532                   "err_warn_bitmap=%08x:%08x\n",
1533                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1534                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1535         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1536                   "rx_buff_off=%08x, rx_id=%04x\n",
1537                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1538                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1539                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1540
1541         if (qedf->stop_io_on_error) {
1542                 qedf_stop_all_io(qedf);
1543                 return;
1544         }
1545
1546         init_completion(&io_req->abts_done);
1547         rval = qedf_initiate_abts(io_req, true);
1548         if (rval)
1549                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1550 }
1551
1552 static void qedf_flush_els_req(struct qedf_ctx *qedf,
1553         struct qedf_ioreq *els_req)
1554 {
1555         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1556             "Flushing ELS request xid=0x%x refcount=%d.\n", els_req->xid,
1557             kref_read(&els_req->refcount));
1558
1559         /*
1560          * Need to distinguish this from a timeout when calling the
1561          * els_req->cb_func.
1562          */
1563         els_req->event = QEDF_IOREQ_EV_ELS_FLUSH;
1564
1565         clear_bit(QEDF_CMD_OUTSTANDING, &els_req->flags);
1566
1567         /* Cancel the timer */
1568         cancel_delayed_work_sync(&els_req->timeout_work);
1569
1570         /* Call callback function to complete command */
1571         if (els_req->cb_func && els_req->cb_arg) {
1572                 els_req->cb_func(els_req->cb_arg);
1573                 els_req->cb_arg = NULL;
1574         }
1575
1576         /* Release kref for original initiate_els */
1577         kref_put(&els_req->refcount, qedf_release_cmd);
1578 }
1579
1580 /* A value of -1 for lun is a wild card that means flush all
1581  * active SCSI I/Os for the target.
1582  */
1583 void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
1584 {
1585         struct qedf_ioreq *io_req;
1586         struct qedf_ctx *qedf;
1587         struct qedf_cmd_mgr *cmd_mgr;
1588         int i, rc;
1589         unsigned long flags;
1590         int flush_cnt = 0;
1591         int wait_cnt = 100;
1592         int refcount = 0;
1593
1594         if (!fcport) {
1595                 QEDF_ERR(NULL, "fcport is NULL\n");
1596                 return;
1597         }
1598
1599         /* Check that fcport is still offloaded */
1600         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1601                 QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
1602                 return;
1603         }
1604
1605         qedf = fcport->qedf;
1606
1607         if (!qedf) {
1608                 QEDF_ERR(NULL, "qedf is NULL.\n");
1609                 return;
1610         }
1611
1612         /* Only wait for all commands to be queued in the Upload context */
1613         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
1614             (lun == -1)) {
1615                 while (atomic_read(&fcport->ios_to_queue)) {
1616                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1617                                   "Waiting for %d I/Os to be queued\n",
1618                                   atomic_read(&fcport->ios_to_queue));
1619                         if (wait_cnt == 0) {
1620                                 QEDF_ERR(NULL,
1621                                          "%d IOs request could not be queued\n",
1622                                          atomic_read(&fcport->ios_to_queue));
1623                         }
1624                         msleep(20);
1625                         wait_cnt--;
1626                 }
1627         }
1628
1629         cmd_mgr = qedf->cmd_mgr;
1630
1631         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1632                   "Flush active i/o's num=0x%x fcport=0x%p port_id=0x%06x scsi_id=%d.\n",
1633                   atomic_read(&fcport->num_active_ios), fcport,
1634                   fcport->rdata->ids.port_id, fcport->rport->scsi_target_id);
1635         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Locking flush mutex.\n");
1636
1637         mutex_lock(&qedf->flush_mutex);
1638         if (lun == -1) {
1639                 set_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
1640         } else {
1641                 set_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
1642                 fcport->lun_reset_lun = lun;
1643         }
1644
1645         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1646                 io_req = &cmd_mgr->cmds[i];
1647
1648                 if (!io_req)
1649                         continue;
1650                 if (!io_req->fcport)
1651                         continue;
1652
1653                 spin_lock_irqsave(&cmd_mgr->lock, flags);
1654
1655                 if (io_req->alloc) {
1656                         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
1657                                 if (io_req->cmd_type == QEDF_SCSI_CMD)
1658                                         QEDF_ERR(&qedf->dbg_ctx,
1659                                                  "Allocated but not queued, xid=0x%x\n",
1660                                                  io_req->xid);
1661                         }
1662                         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
1663                 } else {
1664                         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
1665                         continue;
1666                 }
1667
1668                 if (io_req->fcport != fcport)
1669                         continue;
1670
1671                 /* In case of ABTS, CMD_OUTSTANDING is cleared on ABTS response,
1672                  * but RRQ is still pending.
1673                  * Workaround: Within qedf_send_rrq, we check if the fcport is
1674                  * NULL, and we drop the ref on the io_req to clean it up.
1675                  */
1676                 if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
1677                         refcount = kref_read(&io_req->refcount);
1678                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1679                                   "Not outstanding, xid=0x%x, cmd_type=%d refcount=%d.\n",
1680                                   io_req->xid, io_req->cmd_type, refcount);
1681                         /* If RRQ work has been queue, try to cancel it and
1682                          * free the io_req
1683                          */
1684                         if (atomic_read(&io_req->state) ==
1685                             QEDFC_CMD_ST_RRQ_WAIT) {
1686                                 if (cancel_delayed_work_sync
1687                                     (&io_req->rrq_work)) {
1688                                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1689                                                   "Putting reference for pending RRQ work xid=0x%x.\n",
1690                                                   io_req->xid);
1691                                         /* ID: 003 */
1692                                         kref_put(&io_req->refcount,
1693                                                  qedf_release_cmd);
1694                                 }
1695                         }
1696                         continue;
1697                 }
1698
1699                 /* Only consider flushing ELS during target reset */
1700                 if (io_req->cmd_type == QEDF_ELS &&
1701                     lun == -1) {
1702                         rc = kref_get_unless_zero(&io_req->refcount);
1703                         if (!rc) {
1704                                 QEDF_ERR(&(qedf->dbg_ctx),
1705                                     "Could not get kref for ELS io_req=0x%p xid=0x%x.\n",
1706                                     io_req, io_req->xid);
1707                                 continue;
1708                         }
1709                         qedf_initiate_cleanup(io_req, false);
1710                         flush_cnt++;
1711                         qedf_flush_els_req(qedf, io_req);
1712
1713                         /*
1714                          * Release the kref and go back to the top of the
1715                          * loop.
1716                          */
1717                         goto free_cmd;
1718                 }
1719
1720                 if (io_req->cmd_type == QEDF_ABTS) {
1721                         /* ID: 004 */
1722                         rc = kref_get_unless_zero(&io_req->refcount);
1723                         if (!rc) {
1724                                 QEDF_ERR(&(qedf->dbg_ctx),
1725                                     "Could not get kref for abort io_req=0x%p xid=0x%x.\n",
1726                                     io_req, io_req->xid);
1727                                 continue;
1728                         }
1729                         if (lun != -1 && io_req->lun != lun)
1730                                 goto free_cmd;
1731
1732                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1733                             "Flushing abort xid=0x%x.\n", io_req->xid);
1734
1735                         if (cancel_delayed_work_sync(&io_req->rrq_work)) {
1736                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1737                                           "Putting ref for cancelled RRQ work xid=0x%x.\n",
1738                                           io_req->xid);
1739                                 kref_put(&io_req->refcount, qedf_release_cmd);
1740                         }
1741
1742                         if (cancel_delayed_work_sync(&io_req->timeout_work)) {
1743                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1744                                           "Putting ref for cancelled tmo work xid=0x%x.\n",
1745                                           io_req->xid);
1746                                 qedf_initiate_cleanup(io_req, true);
1747                                 /* Notify eh_abort handler that ABTS is
1748                                  * complete
1749                                  */
1750                                 complete(&io_req->abts_done);
1751                                 clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1752                                 /* ID: 002 */
1753                                 kref_put(&io_req->refcount, qedf_release_cmd);
1754                         }
1755                         flush_cnt++;
1756                         goto free_cmd;
1757                 }
1758
1759                 if (!io_req->sc_cmd)
1760                         continue;
1761                 if (!io_req->sc_cmd->device) {
1762                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1763                                   "Device backpointer NULL for sc_cmd=%p.\n",
1764                                   io_req->sc_cmd);
1765                         /* Put reference for non-existent scsi_cmnd */
1766                         io_req->sc_cmd = NULL;
1767                         qedf_initiate_cleanup(io_req, false);
1768                         kref_put(&io_req->refcount, qedf_release_cmd);
1769                         continue;
1770                 }
1771                 if (lun > -1) {
1772                         if (io_req->lun != lun)
1773                                 continue;
1774                 }
1775
1776                 /*
1777                  * Use kref_get_unless_zero in the unlikely case the command
1778                  * we're about to flush was completed in the normal SCSI path
1779                  */
1780                 rc = kref_get_unless_zero(&io_req->refcount);
1781                 if (!rc) {
1782                         QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
1783                             "io_req=0x%p xid=0x%x\n", io_req, io_req->xid);
1784                         continue;
1785                 }
1786
1787                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1788                     "Cleanup xid=0x%x.\n", io_req->xid);
1789                 flush_cnt++;
1790
1791                 /* Cleanup task and return I/O mid-layer */
1792                 qedf_initiate_cleanup(io_req, true);
1793
1794 free_cmd:
1795                 kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 004 */
1796         }
1797
1798         wait_cnt = 60;
1799         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1800                   "Flushed 0x%x I/Os, active=0x%x.\n",
1801                   flush_cnt, atomic_read(&fcport->num_active_ios));
1802         /* Only wait for all commands to complete in the Upload context */
1803         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
1804             (lun == -1)) {
1805                 while (atomic_read(&fcport->num_active_ios)) {
1806                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1807                                   "Flushed 0x%x I/Os, active=0x%x cnt=%d.\n",
1808                                   flush_cnt,
1809                                   atomic_read(&fcport->num_active_ios),
1810                                   wait_cnt);
1811                         if (wait_cnt == 0) {
1812                                 QEDF_ERR(&qedf->dbg_ctx,
1813                                          "Flushed %d I/Os, active=%d.\n",
1814                                          flush_cnt,
1815                                          atomic_read(&fcport->num_active_ios));
1816                                 for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1817                                         io_req = &cmd_mgr->cmds[i];
1818                                         if (io_req->fcport &&
1819                                             io_req->fcport == fcport) {
1820                                                 refcount =
1821                                                 kref_read(&io_req->refcount);
1822                                                 set_bit(QEDF_CMD_DIRTY,
1823                                                         &io_req->flags);
1824                                                 QEDF_ERR(&qedf->dbg_ctx,
1825                                                          "Outstanding io_req =%p xid=0x%x flags=0x%lx, sc_cmd=%p refcount=%d cmd_type=%d.\n",
1826                                                          io_req, io_req->xid,
1827                                                          io_req->flags,
1828                                                          io_req->sc_cmd,
1829                                                          refcount,
1830                                                          io_req->cmd_type);
1831                                         }
1832                                 }
1833                                 WARN_ON(1);
1834                                 break;
1835                         }
1836                         msleep(500);
1837                         wait_cnt--;
1838                 }
1839         }
1840
1841         clear_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
1842         clear_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
1843         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Unlocking flush mutex.\n");
1844         mutex_unlock(&qedf->flush_mutex);
1845 }
1846
1847 /*
1848  * Initiate a ABTS middle path command. Note that we don't have to initialize
1849  * the task context for an ABTS task.
1850  */
1851 int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
1852 {
1853         struct fc_lport *lport;
1854         struct qedf_rport *fcport = io_req->fcport;
1855         struct fc_rport_priv *rdata;
1856         struct qedf_ctx *qedf;
1857         u16 xid;
1858         int rc = 0;
1859         unsigned long flags;
1860         struct fcoe_wqe *sqe;
1861         u16 sqe_idx;
1862         int refcount = 0;
1863
1864         /* Sanity check qedf_rport before dereferencing any pointers */
1865         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1866                 QEDF_ERR(NULL, "tgt not offloaded\n");
1867                 rc = 1;
1868                 goto out;
1869         }
1870
1871         qedf = fcport->qedf;
1872         rdata = fcport->rdata;
1873
1874         if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
1875                 QEDF_ERR(&qedf->dbg_ctx, "stale rport\n");
1876                 rc = 1;
1877                 goto out;
1878         }
1879
1880         lport = qedf->lport;
1881
1882         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
1883                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
1884                 rc = 1;
1885                 goto drop_rdata_kref;
1886         }
1887
1888         if (atomic_read(&qedf->link_down_tmo_valid) > 0) {
1889                 QEDF_ERR(&(qedf->dbg_ctx), "link_down_tmo active.\n");
1890                 rc = 1;
1891                 goto drop_rdata_kref;
1892         }
1893
1894         /* Ensure room on SQ */
1895         if (!atomic_read(&fcport->free_sqes)) {
1896                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
1897                 rc = 1;
1898                 goto drop_rdata_kref;
1899         }
1900
1901         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
1902                 QEDF_ERR(&qedf->dbg_ctx, "fcport is uploading.\n");
1903                 rc = 1;
1904                 goto drop_rdata_kref;
1905         }
1906
1907         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1908             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
1909             test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
1910                 QEDF_ERR(&qedf->dbg_ctx,
1911                          "io_req xid=0x%x sc_cmd=%p already in cleanup or abort processing or already completed.\n",
1912                          io_req->xid, io_req->sc_cmd);
1913                 rc = 1;
1914                 goto drop_rdata_kref;
1915         }
1916
1917         kref_get(&io_req->refcount);
1918
1919         xid = io_req->xid;
1920         qedf->control_requests++;
1921         qedf->packet_aborts++;
1922
1923         /* Set the command type to abort */
1924         io_req->cmd_type = QEDF_ABTS;
1925         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
1926
1927         set_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1928         refcount = kref_read(&io_req->refcount);
1929         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
1930                   "ABTS io_req xid = 0x%x refcount=%d\n",
1931                   xid, refcount);
1932
1933         qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT);
1934
1935         spin_lock_irqsave(&fcport->rport_lock, flags);
1936
1937         sqe_idx = qedf_get_sqe_idx(fcport);
1938         sqe = &fcport->sq[sqe_idx];
1939         memset(sqe, 0, sizeof(struct fcoe_wqe));
1940         io_req->task_params->sqe = sqe;
1941
1942         init_initiator_abort_fcoe_task(io_req->task_params);
1943         qedf_ring_doorbell(fcport);
1944
1945         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1946
1947 drop_rdata_kref:
1948         kref_put(&rdata->kref, fc_rport_destroy);
1949 out:
1950         return rc;
1951 }
1952
1953 void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1954         struct qedf_ioreq *io_req)
1955 {
1956         uint32_t r_ctl;
1957         int rc;
1958         struct qedf_rport *fcport = io_req->fcport;
1959
1960         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
1961                    "0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
1962
1963         r_ctl = cqe->cqe_info.abts_info.r_ctl;
1964
1965         /* This was added at a point when we were scheduling abts_compl &
1966          * cleanup_compl on different CPUs and there was a possibility of
1967          * the io_req to be freed from the other context before we got here.
1968          */
1969         if (!fcport) {
1970                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1971                           "Dropping ABTS completion xid=0x%x as fcport is NULL",
1972                           io_req->xid);
1973                 return;
1974         }
1975
1976         /*
1977          * When flush is active, let the cmds be completed from the cleanup
1978          * context
1979          */
1980         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
1981             test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags)) {
1982                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1983                           "Dropping ABTS completion xid=0x%x as fcport is flushing",
1984                           io_req->xid);
1985                 return;
1986         }
1987
1988         if (!cancel_delayed_work(&io_req->timeout_work)) {
1989                 QEDF_ERR(&qedf->dbg_ctx,
1990                          "Wasn't able to cancel abts timeout work.\n");
1991         }
1992
1993         switch (r_ctl) {
1994         case FC_RCTL_BA_ACC:
1995                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
1996                     "ABTS response - ACC Send RRQ after R_A_TOV\n");
1997                 io_req->event = QEDF_IOREQ_EV_ABORT_SUCCESS;
1998                 rc = kref_get_unless_zero(&io_req->refcount);   /* ID: 003 */
1999                 if (!rc) {
2000                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2001                                   "kref is already zero so ABTS was already completed or flushed xid=0x%x.\n",
2002                                   io_req->xid);
2003                         return;
2004                 }
2005                 /*
2006                  * Dont release this cmd yet. It will be relesed
2007                  * after we get RRQ response
2008                  */
2009                 queue_delayed_work(qedf->dpc_wq, &io_req->rrq_work,
2010                     msecs_to_jiffies(qedf->lport->r_a_tov));
2011                 atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_WAIT);
2012                 break;
2013         /* For error cases let the cleanup return the command */
2014         case FC_RCTL_BA_RJT:
2015                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
2016                    "ABTS response - RJT\n");
2017                 io_req->event = QEDF_IOREQ_EV_ABORT_FAILED;
2018                 break;
2019         default:
2020                 QEDF_ERR(&(qedf->dbg_ctx), "Unknown ABTS response\n");
2021                 break;
2022         }
2023
2024         clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
2025
2026         if (io_req->sc_cmd) {
2027                 if (!io_req->return_scsi_cmd_on_abts)
2028                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2029                                   "Not call scsi_done for xid=0x%x.\n",
2030                                   io_req->xid);
2031                 if (io_req->return_scsi_cmd_on_abts)
2032                         qedf_scsi_done(qedf, io_req, DID_ERROR);
2033         }
2034
2035         /* Notify eh_abort handler that ABTS is complete */
2036         complete(&io_req->abts_done);
2037
2038         kref_put(&io_req->refcount, qedf_release_cmd);
2039 }
2040
2041 int qedf_init_mp_req(struct qedf_ioreq *io_req)
2042 {
2043         struct qedf_mp_req *mp_req;
2044         struct scsi_sge *mp_req_bd;
2045         struct scsi_sge *mp_resp_bd;
2046         struct qedf_ctx *qedf = io_req->fcport->qedf;
2047         dma_addr_t addr;
2048         uint64_t sz;
2049
2050         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_MP_REQ, "Entered.\n");
2051
2052         mp_req = (struct qedf_mp_req *)&(io_req->mp_req);
2053         memset(mp_req, 0, sizeof(struct qedf_mp_req));
2054
2055         if (io_req->cmd_type != QEDF_ELS) {
2056                 mp_req->req_len = sizeof(struct fcp_cmnd);
2057                 io_req->data_xfer_len = mp_req->req_len;
2058         } else
2059                 mp_req->req_len = io_req->data_xfer_len;
2060
2061         mp_req->req_buf = dma_alloc_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
2062             &mp_req->req_buf_dma, GFP_KERNEL);
2063         if (!mp_req->req_buf) {
2064                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req buffer\n");
2065                 qedf_free_mp_resc(io_req);
2066                 return -ENOMEM;
2067         }
2068
2069         mp_req->resp_buf = dma_alloc_coherent(&qedf->pdev->dev,
2070             QEDF_PAGE_SIZE, &mp_req->resp_buf_dma, GFP_KERNEL);
2071         if (!mp_req->resp_buf) {
2072                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc TM resp "
2073                           "buffer\n");
2074                 qedf_free_mp_resc(io_req);
2075                 return -ENOMEM;
2076         }
2077
2078         /* Allocate and map mp_req_bd and mp_resp_bd */
2079         sz = sizeof(struct scsi_sge);
2080         mp_req->mp_req_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
2081             &mp_req->mp_req_bd_dma, GFP_KERNEL);
2082         if (!mp_req->mp_req_bd) {
2083                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req bd\n");
2084                 qedf_free_mp_resc(io_req);
2085                 return -ENOMEM;
2086         }
2087
2088         mp_req->mp_resp_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
2089             &mp_req->mp_resp_bd_dma, GFP_KERNEL);
2090         if (!mp_req->mp_resp_bd) {
2091                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP resp bd\n");
2092                 qedf_free_mp_resc(io_req);
2093                 return -ENOMEM;
2094         }
2095
2096         /* Fill bd table */
2097         addr = mp_req->req_buf_dma;
2098         mp_req_bd = mp_req->mp_req_bd;
2099         mp_req_bd->sge_addr.lo = U64_LO(addr);
2100         mp_req_bd->sge_addr.hi = U64_HI(addr);
2101         mp_req_bd->sge_len = QEDF_PAGE_SIZE;
2102
2103         /*
2104          * MP buffer is either a task mgmt command or an ELS.
2105          * So the assumption is that it consumes a single bd
2106          * entry in the bd table
2107          */
2108         mp_resp_bd = mp_req->mp_resp_bd;
2109         addr = mp_req->resp_buf_dma;
2110         mp_resp_bd->sge_addr.lo = U64_LO(addr);
2111         mp_resp_bd->sge_addr.hi = U64_HI(addr);
2112         mp_resp_bd->sge_len = QEDF_PAGE_SIZE;
2113
2114         return 0;
2115 }
2116
2117 /*
2118  * Last ditch effort to clear the port if it's stuck. Used only after a
2119  * cleanup task times out.
2120  */
2121 static void qedf_drain_request(struct qedf_ctx *qedf)
2122 {
2123         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
2124                 QEDF_ERR(&(qedf->dbg_ctx), "MCP drain already active.\n");
2125                 return;
2126         }
2127
2128         /* Set bit to return all queuecommand requests as busy */
2129         set_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
2130
2131         /* Call qed drain request for function. Should be synchronous */
2132         qed_ops->common->drain(qedf->cdev);
2133
2134         /* Settle time for CQEs to be returned */
2135         msleep(100);
2136
2137         /* Unplug and continue */
2138         clear_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
2139 }
2140
2141 /*
2142  * Returns SUCCESS if the cleanup task does not timeout, otherwise return
2143  * FAILURE.
2144  */
2145 int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
2146         bool return_scsi_cmd_on_abts)
2147 {
2148         struct qedf_rport *fcport;
2149         struct qedf_ctx *qedf;
2150         int tmo = 0;
2151         int rc = SUCCESS;
2152         unsigned long flags;
2153         struct fcoe_wqe *sqe;
2154         u16 sqe_idx;
2155         int refcount = 0;
2156
2157         fcport = io_req->fcport;
2158         if (!fcport) {
2159                 QEDF_ERR(NULL, "fcport is NULL.\n");
2160                 return SUCCESS;
2161         }
2162
2163         /* Sanity check qedf_rport before dereferencing any pointers */
2164         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
2165                 QEDF_ERR(NULL, "tgt not offloaded\n");
2166                 return SUCCESS;
2167         }
2168
2169         qedf = fcport->qedf;
2170         if (!qedf) {
2171                 QEDF_ERR(NULL, "qedf is NULL.\n");
2172                 return SUCCESS;
2173         }
2174
2175         if (io_req->cmd_type == QEDF_ELS) {
2176                 goto process_els;
2177         }
2178
2179         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
2180             test_and_set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
2181                 QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
2182                           "cleanup processing or already completed.\n",
2183                           io_req->xid);
2184                 return SUCCESS;
2185         }
2186         set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2187
2188 process_els:
2189         /* Ensure room on SQ */
2190         if (!atomic_read(&fcport->free_sqes)) {
2191                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
2192                 /* Need to make sure we clear the flag since it was set */
2193                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2194                 return FAILED;
2195         }
2196
2197         if (io_req->cmd_type == QEDF_CLEANUP) {
2198                 QEDF_ERR(&qedf->dbg_ctx,
2199                          "io_req=0x%x is already a cleanup command cmd_type=%d.\n",
2200                          io_req->xid, io_req->cmd_type);
2201                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2202                 return SUCCESS;
2203         }
2204
2205         refcount = kref_read(&io_req->refcount);
2206
2207         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
2208                   "Entered xid=0x%x sc_cmd=%p cmd_type=%d flags=0x%lx refcount=%d fcport=%p port_id=0x%06x\n",
2209                   io_req->xid, io_req->sc_cmd, io_req->cmd_type, io_req->flags,
2210                   refcount, fcport, fcport->rdata->ids.port_id);
2211
2212         /* Cleanup cmds re-use the same TID as the original I/O */
2213         io_req->cmd_type = QEDF_CLEANUP;
2214         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
2215
2216         init_completion(&io_req->cleanup_done);
2217
2218         spin_lock_irqsave(&fcport->rport_lock, flags);
2219
2220         sqe_idx = qedf_get_sqe_idx(fcport);
2221         sqe = &fcport->sq[sqe_idx];
2222         memset(sqe, 0, sizeof(struct fcoe_wqe));
2223         io_req->task_params->sqe = sqe;
2224
2225         init_initiator_cleanup_fcoe_task(io_req->task_params);
2226         qedf_ring_doorbell(fcport);
2227
2228         spin_unlock_irqrestore(&fcport->rport_lock, flags);
2229
2230         tmo = wait_for_completion_timeout(&io_req->cleanup_done,
2231                                           QEDF_CLEANUP_TIMEOUT * HZ);
2232
2233         if (!tmo) {
2234                 rc = FAILED;
2235                 /* Timeout case */
2236                 QEDF_ERR(&(qedf->dbg_ctx), "Cleanup command timeout, "
2237                           "xid=%x.\n", io_req->xid);
2238                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2239                 /* Issue a drain request if cleanup task times out */
2240                 QEDF_ERR(&(qedf->dbg_ctx), "Issuing MCP drain request.\n");
2241                 qedf_drain_request(qedf);
2242         }
2243
2244         /* If it TASK MGMT handle it, reference will be decreased
2245          * in qedf_execute_tmf
2246          */
2247         if (io_req->tm_flags  == FCP_TMF_LUN_RESET ||
2248             io_req->tm_flags == FCP_TMF_TGT_RESET) {
2249                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2250                 io_req->sc_cmd = NULL;
2251                 complete(&io_req->tm_done);
2252         }
2253
2254         if (io_req->sc_cmd) {
2255                 if (!io_req->return_scsi_cmd_on_abts)
2256                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2257                                   "Not call scsi_done for xid=0x%x.\n",
2258                                   io_req->xid);
2259                 if (io_req->return_scsi_cmd_on_abts)
2260                         qedf_scsi_done(qedf, io_req, DID_ERROR);
2261         }
2262
2263         if (rc == SUCCESS)
2264                 io_req->event = QEDF_IOREQ_EV_CLEANUP_SUCCESS;
2265         else
2266                 io_req->event = QEDF_IOREQ_EV_CLEANUP_FAILED;
2267
2268         return rc;
2269 }
2270
2271 void qedf_process_cleanup_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
2272         struct qedf_ioreq *io_req)
2273 {
2274         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid = 0x%x\n",
2275                    io_req->xid);
2276
2277         clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2278
2279         /* Complete so we can finish cleaning up the I/O */
2280         complete(&io_req->cleanup_done);
2281 }
2282
2283 static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
2284         uint8_t tm_flags)
2285 {
2286         struct qedf_ioreq *io_req;
2287         struct e4_fcoe_task_context *task;
2288         struct qedf_ctx *qedf = fcport->qedf;
2289         struct fc_lport *lport = qedf->lport;
2290         int rc = 0;
2291         uint16_t xid;
2292         int tmo = 0;
2293         int lun = 0;
2294         unsigned long flags;
2295         struct fcoe_wqe *sqe;
2296         u16 sqe_idx;
2297
2298         if (!sc_cmd) {
2299                 QEDF_ERR(&qedf->dbg_ctx, "sc_cmd is NULL\n");
2300                 return FAILED;
2301         }
2302
2303         lun = (int)sc_cmd->device->lun;
2304         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
2305                 QEDF_ERR(&(qedf->dbg_ctx), "fcport not offloaded\n");
2306                 rc = FAILED;
2307                 goto no_flush;
2308         }
2309
2310         io_req = qedf_alloc_cmd(fcport, QEDF_TASK_MGMT_CMD);
2311         if (!io_req) {
2312                 QEDF_ERR(&(qedf->dbg_ctx), "Failed TMF");
2313                 rc = -EAGAIN;
2314                 goto no_flush;
2315         }
2316
2317         if (tm_flags == FCP_TMF_LUN_RESET)
2318                 qedf->lun_resets++;
2319         else if (tm_flags == FCP_TMF_TGT_RESET)
2320                 qedf->target_resets++;
2321
2322         /* Initialize rest of io_req fields */
2323         io_req->sc_cmd = sc_cmd;
2324         io_req->fcport = fcport;
2325         io_req->cmd_type = QEDF_TASK_MGMT_CMD;
2326
2327         /* Record which cpu this request is associated with */
2328         io_req->cpu = smp_processor_id();
2329
2330         /* Set TM flags */
2331         io_req->io_req_flags = QEDF_READ;
2332         io_req->data_xfer_len = 0;
2333         io_req->tm_flags = tm_flags;
2334
2335         /* Default is to return a SCSI command when an error occurs */
2336         io_req->return_scsi_cmd_on_abts = false;
2337
2338         /* Obtain exchange id */
2339         xid = io_req->xid;
2340
2341         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "TMF io_req xid = "
2342                    "0x%x\n", xid);
2343
2344         /* Initialize task context for this IO request */
2345         task = qedf_get_task_mem(&qedf->tasks, xid);
2346
2347         init_completion(&io_req->tm_done);
2348
2349         spin_lock_irqsave(&fcport->rport_lock, flags);
2350
2351         sqe_idx = qedf_get_sqe_idx(fcport);
2352         sqe = &fcport->sq[sqe_idx];
2353         memset(sqe, 0, sizeof(struct fcoe_wqe));
2354
2355         qedf_init_task(fcport, lport, io_req, task, sqe);
2356         qedf_ring_doorbell(fcport);
2357
2358         spin_unlock_irqrestore(&fcport->rport_lock, flags);
2359
2360         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2361         tmo = wait_for_completion_timeout(&io_req->tm_done,
2362             QEDF_TM_TIMEOUT * HZ);
2363
2364         if (!tmo) {
2365                 rc = FAILED;
2366                 QEDF_ERR(&(qedf->dbg_ctx), "wait for tm_cmpl timeout!\n");
2367                 /* Clear outstanding bit since command timed out */
2368                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2369                 io_req->sc_cmd = NULL;
2370         } else {
2371                 /* Check TMF response code */
2372                 if (io_req->fcp_rsp_code == 0)
2373                         rc = SUCCESS;
2374                 else
2375                         rc = FAILED;
2376         }
2377         /*
2378          * Double check that fcport has not gone into an uploading state before
2379          * executing the command flush for the LUN/target.
2380          */
2381         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2382                 QEDF_ERR(&qedf->dbg_ctx,
2383                          "fcport is uploading, not executing flush.\n");
2384                 goto no_flush;
2385         }
2386         /* We do not need this io_req any more */
2387         kref_put(&io_req->refcount, qedf_release_cmd);
2388
2389
2390         if (tm_flags == FCP_TMF_LUN_RESET)
2391                 qedf_flush_active_ios(fcport, lun);
2392         else
2393                 qedf_flush_active_ios(fcport, -1);
2394
2395 no_flush:
2396         if (rc != SUCCESS) {
2397                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command failed...\n");
2398                 rc = FAILED;
2399         } else {
2400                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command success...\n");
2401                 rc = SUCCESS;
2402         }
2403         return rc;
2404 }
2405
2406 int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
2407 {
2408         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
2409         struct fc_rport_libfc_priv *rp = rport->dd_data;
2410         struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
2411         struct qedf_ctx *qedf;
2412         struct fc_lport *lport = shost_priv(sc_cmd->device->host);
2413         int rc = SUCCESS;
2414         int rval;
2415         struct qedf_ioreq *io_req = NULL;
2416         int ref_cnt = 0;
2417         struct fc_rport_priv *rdata = fcport->rdata;
2418
2419         QEDF_ERR(NULL,
2420                  "tm_flags 0x%x sc_cmd %p op = 0x%02x target_id = 0x%x lun=%d\n",
2421                  tm_flags, sc_cmd, sc_cmd->cmd_len ? sc_cmd->cmnd[0] : 0xff,
2422                  rport->scsi_target_id, (int)sc_cmd->device->lun);
2423
2424         if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
2425                 QEDF_ERR(NULL, "stale rport\n");
2426                 return FAILED;
2427         }
2428
2429         QEDF_ERR(NULL, "portid=%06x tm_flags =%s\n", rdata->ids.port_id,
2430                  (tm_flags == FCP_TMF_TGT_RESET) ? "TARGET RESET" :
2431                  "LUN RESET");
2432
2433         if (sc_cmd->SCp.ptr) {
2434                 io_req = (struct qedf_ioreq *)sc_cmd->SCp.ptr;
2435                 ref_cnt = kref_read(&io_req->refcount);
2436                 QEDF_ERR(NULL,
2437                          "orig io_req = %p xid = 0x%x ref_cnt = %d.\n",
2438                          io_req, io_req->xid, ref_cnt);
2439         }
2440
2441         rval = fc_remote_port_chkready(rport);
2442         if (rval) {
2443                 QEDF_ERR(NULL, "device_reset rport not ready\n");
2444                 rc = FAILED;
2445                 goto tmf_err;
2446         }
2447
2448         rc = fc_block_scsi_eh(sc_cmd);
2449         if (rc)
2450                 goto tmf_err;
2451
2452         if (!fcport) {
2453                 QEDF_ERR(NULL, "device_reset: rport is NULL\n");
2454                 rc = FAILED;
2455                 goto tmf_err;
2456         }
2457
2458         qedf = fcport->qedf;
2459
2460         if (!qedf) {
2461                 QEDF_ERR(NULL, "qedf is NULL.\n");
2462                 rc = FAILED;
2463                 goto tmf_err;
2464         }
2465
2466         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2467                 QEDF_ERR(&qedf->dbg_ctx, "Connection is getting uploaded.\n");
2468                 rc = SUCCESS;
2469                 goto tmf_err;
2470         }
2471
2472         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
2473             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
2474                 rc = SUCCESS;
2475                 goto tmf_err;
2476         }
2477
2478         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
2479                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
2480                 rc = FAILED;
2481                 goto tmf_err;
2482         }
2483
2484         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2485                 if (!fcport->rdata)
2486                         QEDF_ERR(&qedf->dbg_ctx, "fcport %p is uploading.\n",
2487                                  fcport);
2488                 else
2489                         QEDF_ERR(&qedf->dbg_ctx,
2490                                  "fcport %p port_id=%06x is uploading.\n",
2491                                  fcport, fcport->rdata->ids.port_id);
2492                 rc = FAILED;
2493                 goto tmf_err;
2494         }
2495
2496         rc = qedf_execute_tmf(fcport, sc_cmd, tm_flags);
2497
2498 tmf_err:
2499         kref_put(&rdata->kref, fc_rport_destroy);
2500         return rc;
2501 }
2502
2503 void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
2504         struct qedf_ioreq *io_req)
2505 {
2506         struct fcoe_cqe_rsp_info *fcp_rsp;
2507
2508         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2509
2510         fcp_rsp = &cqe->cqe_info.rsp_info;
2511         qedf_parse_fcp_rsp(io_req, fcp_rsp);
2512
2513         io_req->sc_cmd = NULL;
2514         complete(&io_req->tm_done);
2515 }
2516
2517 void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
2518         struct fcoe_cqe *cqe)
2519 {
2520         unsigned long flags;
2521         uint16_t pktlen = cqe->cqe_info.unsolic_info.pkt_len;
2522         u32 payload_len, crc;
2523         struct fc_frame_header *fh;
2524         struct fc_frame *fp;
2525         struct qedf_io_work *io_work;
2526         u32 bdq_idx;
2527         void *bdq_addr;
2528         struct scsi_bd *p_bd_info;
2529
2530         p_bd_info = &cqe->cqe_info.unsolic_info.bd_info;
2531         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2532                   "address.hi=%x, address.lo=%x, opaque_data.hi=%x, opaque_data.lo=%x, bdq_prod_idx=%u, len=%u\n",
2533                   le32_to_cpu(p_bd_info->address.hi),
2534                   le32_to_cpu(p_bd_info->address.lo),
2535                   le32_to_cpu(p_bd_info->opaque.fcoe_opaque.hi),
2536                   le32_to_cpu(p_bd_info->opaque.fcoe_opaque.lo),
2537                   qedf->bdq_prod_idx, pktlen);
2538
2539         bdq_idx = le32_to_cpu(p_bd_info->opaque.fcoe_opaque.lo);
2540         if (bdq_idx >= QEDF_BDQ_SIZE) {
2541                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_idx is out of range %d.\n",
2542                     bdq_idx);
2543                 goto increment_prod;
2544         }
2545
2546         bdq_addr = qedf->bdq[bdq_idx].buf_addr;
2547         if (!bdq_addr) {
2548                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_addr is NULL, dropping "
2549                     "unsolicited packet.\n");
2550                 goto increment_prod;
2551         }
2552
2553         if (qedf_dump_frames) {
2554                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2555                     "BDQ frame is at addr=%p.\n", bdq_addr);
2556                 print_hex_dump(KERN_WARNING, "bdq ", DUMP_PREFIX_OFFSET, 16, 1,
2557                     (void *)bdq_addr, pktlen, false);
2558         }
2559
2560         /* Allocate frame */
2561         payload_len = pktlen - sizeof(struct fc_frame_header);
2562         fp = fc_frame_alloc(qedf->lport, payload_len);
2563         if (!fp) {
2564                 QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate fp.\n");
2565                 goto increment_prod;
2566         }
2567
2568         /* Copy data from BDQ buffer into fc_frame struct */
2569         fh = (struct fc_frame_header *)fc_frame_header_get(fp);
2570         memcpy(fh, (void *)bdq_addr, pktlen);
2571
2572         QEDF_WARN(&qedf->dbg_ctx,
2573                   "Processing Unsolicated frame, src=%06x dest=%06x r_ctl=0x%x type=0x%x cmd=%02x\n",
2574                   ntoh24(fh->fh_s_id), ntoh24(fh->fh_d_id), fh->fh_r_ctl,
2575                   fh->fh_type, fc_frame_payload_op(fp));
2576
2577         /* Initialize the frame so libfc sees it as a valid frame */
2578         crc = fcoe_fc_crc(fp);
2579         fc_frame_init(fp);
2580         fr_dev(fp) = qedf->lport;
2581         fr_sof(fp) = FC_SOF_I3;
2582         fr_eof(fp) = FC_EOF_T;
2583         fr_crc(fp) = cpu_to_le32(~crc);
2584
2585         /*
2586          * We need to return the frame back up to libfc in a non-atomic
2587          * context
2588          */
2589         io_work = mempool_alloc(qedf->io_mempool, GFP_ATOMIC);
2590         if (!io_work) {
2591                 QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
2592                            "work for I/O completion.\n");
2593                 fc_frame_free(fp);
2594                 goto increment_prod;
2595         }
2596         memset(io_work, 0, sizeof(struct qedf_io_work));
2597
2598         INIT_WORK(&io_work->work, qedf_fp_io_handler);
2599
2600         /* Copy contents of CQE for deferred processing */
2601         memcpy(&io_work->cqe, cqe, sizeof(struct fcoe_cqe));
2602
2603         io_work->qedf = qedf;
2604         io_work->fp = fp;
2605
2606         queue_work_on(smp_processor_id(), qedf_io_wq, &io_work->work);
2607 increment_prod:
2608         spin_lock_irqsave(&qedf->hba_lock, flags);
2609
2610         /* Increment producer to let f/w know we've handled the frame */
2611         qedf->bdq_prod_idx++;
2612
2613         /* Producer index wraps at uint16_t boundary */
2614         if (qedf->bdq_prod_idx == 0xffff)
2615                 qedf->bdq_prod_idx = 0;
2616
2617         writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
2618         readw(qedf->bdq_primary_prod);
2619         writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
2620         readw(qedf->bdq_secondary_prod);
2621
2622         spin_unlock_irqrestore(&qedf->hba_lock, flags);
2623 }