drivers/infiniband/hw/bnxt_re/qplib_fp.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: Fast Path Operators
  37  */
  38
  39 #define dev_fmt(fmt) "QPLIB: " fmt
  40
  41 #include <linux/interrupt.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/sched.h>
  44 #include <linux/slab.h>
  45 #include <linux/pci.h>
  46 #include <linux/delay.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/if_ether.h>
  49
  50 #include "roce_hsi.h"
  51
  52 #include "qplib_res.h"
  53 #include "qplib_rcfw.h"
  54 #include "qplib_sp.h"
  55 #include "qplib_fp.h"
  56
  57 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
  58
  59 static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
  60 {
  61         qp->sq.condition = false;
  62         qp->sq.send_phantom = false;
  63         qp->sq.single = false;
  64 }
  65
  66 /* Flush list */
  67 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
  68 {
  69         struct bnxt_qplib_cq *scq, *rcq;
  70
  71         scq = qp->scq;
  72         rcq = qp->rcq;
  73
  74         if (!qp->sq.flushed) {
  75                 dev_dbg(&scq->hwq.pdev->dev,
  76                         "FP: Adding to SQ Flush list = %p\n", qp);
  77                 bnxt_qplib_cancel_phantom_processing(qp);
  78                 list_add_tail(&qp->sq_flush, &scq->sqf_head);
  79                 qp->sq.flushed = true;
  80         }
  81         if (!qp->srq) {
  82                 if (!qp->rq.flushed) {
  83                         dev_dbg(&rcq->hwq.pdev->dev,
  84                                 "FP: Adding to RQ Flush list = %p\n", qp);
  85                         list_add_tail(&qp->rq_flush, &rcq->rqf_head);
  86                         qp->rq.flushed = true;
  87                 }
  88         }
  89 }
  90
  91 static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp,
  92                                        unsigned long *flags)
  93         __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
  94 {
  95         spin_lock_irqsave(&qp->scq->flush_lock, *flags);
  96         if (qp->scq == qp->rcq)
  97                 __acquire(&qp->rcq->flush_lock);
  98         else
  99                 spin_lock(&qp->rcq->flush_lock);
 100 }
 101
 102 static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp,
 103                                        unsigned long *flags)
 104         __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
 105 {
 106         if (qp->scq == qp->rcq)
 107                 __release(&qp->rcq->flush_lock);
 108         else
 109                 spin_unlock(&qp->rcq->flush_lock);
 110         spin_unlock_irqrestore(&qp->scq->flush_lock, *flags);
 111 }
 112
 113 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
 114 {
 115         unsigned long flags;
 116
 117         bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
 118         __bnxt_qplib_add_flush_qp(qp);
 119         bnxt_qplib_release_cq_flush_locks(qp, &flags);
 120 }
 121
 122 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
 123 {
 124         if (qp->sq.flushed) {
 125                 qp->sq.flushed = false;
 126                 list_del(&qp->sq_flush);
 127         }
 128         if (!qp->srq) {
 129                 if (qp->rq.flushed) {
 130                         qp->rq.flushed = false;
 131                         list_del(&qp->rq_flush);
 132                 }
 133         }
 134 }
 135
 136 void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
 137 {
 138         unsigned long flags;
 139
 140         bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
 141         __clean_cq(qp->scq, (u64)(unsigned long)qp);
 142         qp->sq.hwq.prod = 0;
 143         qp->sq.hwq.cons = 0;
 144         __clean_cq(qp->rcq, (u64)(unsigned long)qp);
 145         qp->rq.hwq.prod = 0;
 146         qp->rq.hwq.cons = 0;
 147
 148         __bnxt_qplib_del_flush_qp(qp);
 149         bnxt_qplib_release_cq_flush_locks(qp, &flags);
 150 }
 151
 152 static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
 153 {
 154         struct bnxt_qplib_nq_work *nq_work =
 155                         container_of(work, struct bnxt_qplib_nq_work, work);
 156
 157         struct bnxt_qplib_cq *cq = nq_work->cq;
 158         struct bnxt_qplib_nq *nq = nq_work->nq;
 159
 160         if (cq && nq) {
 161                 spin_lock_bh(&cq->compl_lock);
 162                 if (atomic_read(&cq->arm_state) && nq->cqn_handler) {
 163                         dev_dbg(&nq->pdev->dev,
 164                                 "%s:Trigger cq  = %p event nq = %p\n",
 165                                 __func__, cq, nq);
 166                         nq->cqn_handler(nq, cq);
 167                 }
 168                 spin_unlock_bh(&cq->compl_lock);
 169         }
 170         kfree(nq_work);
 171 }
 172
 173 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
 174                                        struct bnxt_qplib_qp *qp)
 175 {
 176         struct bnxt_qplib_q *rq = &qp->rq;
 177         struct bnxt_qplib_q *sq = &qp->sq;
 178
 179         if (qp->rq_hdr_buf)
 180                 dma_free_coherent(&res->pdev->dev,
 181                                   rq->max_wqe * qp->rq_hdr_buf_size,
 182                                   qp->rq_hdr_buf, qp->rq_hdr_buf_map);
 183         if (qp->sq_hdr_buf)
 184                 dma_free_coherent(&res->pdev->dev,
 185                                   sq->max_wqe * qp->sq_hdr_buf_size,
 186                                   qp->sq_hdr_buf, qp->sq_hdr_buf_map);
 187         qp->rq_hdr_buf = NULL;
 188         qp->sq_hdr_buf = NULL;
 189         qp->rq_hdr_buf_map = 0;
 190         qp->sq_hdr_buf_map = 0;
 191         qp->sq_hdr_buf_size = 0;
 192         qp->rq_hdr_buf_size = 0;
 193 }
 194
 195 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
 196                                        struct bnxt_qplib_qp *qp)
 197 {
 198         struct bnxt_qplib_q *rq = &qp->rq;
 199         struct bnxt_qplib_q *sq = &qp->sq;
 200         int rc = 0;
 201
 202         if (qp->sq_hdr_buf_size && sq->max_wqe) {
 203                 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
 204                                         sq->max_wqe * qp->sq_hdr_buf_size,
 205                                         &qp->sq_hdr_buf_map, GFP_KERNEL);
 206                 if (!qp->sq_hdr_buf) {
 207                         rc = -ENOMEM;
 208                         dev_err(&res->pdev->dev,
 209                                 "Failed to create sq_hdr_buf\n");
 210                         goto fail;
 211                 }
 212         }
 213
 214         if (qp->rq_hdr_buf_size && rq->max_wqe) {
 215                 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
 216                                                     rq->max_wqe *
 217                                                     qp->rq_hdr_buf_size,
 218                                                     &qp->rq_hdr_buf_map,
 219                                                     GFP_KERNEL);
 220                 if (!qp->rq_hdr_buf) {
 221                         rc = -ENOMEM;
 222                         dev_err(&res->pdev->dev,
 223                                 "Failed to create rq_hdr_buf\n");
 224                         goto fail;
 225                 }
 226         }
 227         return 0;
 228
 229 fail:
 230         bnxt_qplib_free_qp_hdr_buf(res, qp);
 231         return rc;
 232 }
 233
 234 static void clean_nq(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq)
 235 {
 236         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 237         struct nq_base *nqe, **nq_ptr;
 238         int budget = nq->budget;
 239         u32 sw_cons, raw_cons;
 240         uintptr_t q_handle;
 241         u16 type;
 242
 243         spin_lock_bh(&hwq->lock);
 244         /* Service the NQ until empty */
 245         raw_cons = hwq->cons;
 246         while (budget--) {
 247                 sw_cons = HWQ_CMP(raw_cons, hwq);
 248                 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
 249                 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
 250                 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
 251                         break;
 252
 253                 /*
 254                  * The valid test of the entry must be done first before
 255                  * reading any further.
 256                  */
 257                 dma_rmb();
 258
 259                 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
 260                 switch (type) {
 261                 case NQ_BASE_TYPE_CQ_NOTIFICATION:
 262                 {
 263                         struct nq_cn *nqcne = (struct nq_cn *)nqe;
 264
 265                         q_handle = le32_to_cpu(nqcne->cq_handle_low);
 266                         q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
 267                                                      << 32;
 268                         if ((unsigned long)cq == q_handle) {
 269                                 nqcne->cq_handle_low = 0;
 270                                 nqcne->cq_handle_high = 0;
 271                                 cq->cnq_events++;
 272                         }
 273                         break;
 274                 }
 275                 default:
 276                         break;
 277                 }
 278                 raw_cons++;
 279         }
 280         spin_unlock_bh(&hwq->lock);
 281 }
 282
 283 /* Wait for receiving all NQEs for this CQ and clean the NQEs associated with
 284  * this CQ.
 285  */
 286 static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
 287 {
 288         u32 retry_cnt = 100;
 289
 290         while (retry_cnt--) {
 291                 if (cnq_events == cq->cnq_events)
 292                         return;
 293                 usleep_range(50, 100);
 294                 clean_nq(cq->nq, cq);
 295         }
 296 }
 297
 298 static void bnxt_qplib_service_nq(unsigned long data)
 299 {
 300         struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
 301         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 302         int num_srqne_processed = 0;
 303         int num_cqne_processed = 0;
 304         struct bnxt_qplib_cq *cq;
 305         int budget = nq->budget;
 306         u32 sw_cons, raw_cons;
 307         struct nq_base *nqe;
 308         uintptr_t q_handle;
 309         u16 type;
 310
 311         spin_lock_bh(&hwq->lock);
 312         /* Service the NQ until empty */
 313         raw_cons = hwq->cons;
 314         while (budget--) {
 315                 sw_cons = HWQ_CMP(raw_cons, hwq);
 316                 nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
 317                 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
 318                         break;
 319
 320                 /*
 321                  * The valid test of the entry must be done first before
 322                  * reading any further.
 323                  */
 324                 dma_rmb();
 325
 326                 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
 327                 switch (type) {
 328                 case NQ_BASE_TYPE_CQ_NOTIFICATION:
 329                 {
 330                         struct nq_cn *nqcne = (struct nq_cn *)nqe;
 331
 332                         q_handle = le32_to_cpu(nqcne->cq_handle_low);
 333                         q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
 334                                                      << 32;
 335                         cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
 336                         if (!cq)
 337                                 break;
 338                         bnxt_qplib_armen_db(&cq->dbinfo,
 339                                             DBC_DBC_TYPE_CQ_ARMENA);
 340                         spin_lock_bh(&cq->compl_lock);
 341                         atomic_set(&cq->arm_state, 0);
 342                         if (!nq->cqn_handler(nq, (cq)))
 343                                 num_cqne_processed++;
 344                         else
 345                                 dev_warn(&nq->pdev->dev,
 346                                          "cqn - type 0x%x not handled\n", type);
 347                         cq->cnq_events++;
 348                         spin_unlock_bh(&cq->compl_lock);
 349                         break;
 350                 }
 351                 case NQ_BASE_TYPE_SRQ_EVENT:
 352                 {
 353                         struct bnxt_qplib_srq *srq;
 354                         struct nq_srq_event *nqsrqe =
 355                                                 (struct nq_srq_event *)nqe;
 356
 357                         q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
 358                         q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
 359                                      << 32;
 360                         srq = (struct bnxt_qplib_srq *)q_handle;
 361                         bnxt_qplib_armen_db(&srq->dbinfo,
 362                                             DBC_DBC_TYPE_SRQ_ARMENA);
 363                         if (!nq->srqn_handler(nq,
 364                                               (struct bnxt_qplib_srq *)q_handle,
 365                                               nqsrqe->event))
 366                                 num_srqne_processed++;
 367                         else
 368                                 dev_warn(&nq->pdev->dev,
 369                                          "SRQ event 0x%x not handled\n",
 370                                          nqsrqe->event);
 371                         break;
 372                 }
 373                 case NQ_BASE_TYPE_DBQ_EVENT:
 374                         break;
 375                 default:
 376                         dev_warn(&nq->pdev->dev,
 377                                  "nqe with type = 0x%x not handled\n", type);
 378                         break;
 379                 }
 380                 raw_cons++;
 381         }
 382         if (hwq->cons != raw_cons) {
 383                 hwq->cons = raw_cons;
 384                 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
 385         }
 386         spin_unlock_bh(&hwq->lock);
 387 }
 388
 389 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
 390 {
 391         struct bnxt_qplib_nq *nq = dev_instance;
 392         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 393         u32 sw_cons;
 394
 395         /* Prefetch the NQ element */
 396         sw_cons = HWQ_CMP(hwq->cons, hwq);
 397         prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
 398
 399         /* Fan out to CPU affinitized kthreads? */
 400         tasklet_schedule(&nq->nq_tasklet);
 401
 402         return IRQ_HANDLED;
 403 }
 404
 405 void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
 406 {
 407         tasklet_disable(&nq->nq_tasklet);
 408         /* Mask h/w interrupt */
 409         bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, false);
 410         /* Sync with last running IRQ handler */
 411         synchronize_irq(nq->msix_vec);
 412         if (kill)
 413                 tasklet_kill(&nq->nq_tasklet);
 414         if (nq->requested) {
 415                 irq_set_affinity_hint(nq->msix_vec, NULL);
 416                 free_irq(nq->msix_vec, nq);
 417                 nq->requested = false;
 418         }
 419 }
 420
 421 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 422 {
 423         if (nq->cqn_wq) {
 424                 destroy_workqueue(nq->cqn_wq);
 425                 nq->cqn_wq = NULL;
 426         }
 427
 428         /* Make sure the HW is stopped! */
 429         bnxt_qplib_nq_stop_irq(nq, true);
 430
 431         if (nq->nq_db.reg.bar_reg) {
 432                 iounmap(nq->nq_db.reg.bar_reg);
 433                 nq->nq_db.reg.bar_reg = NULL;
 434         }
 435
 436         nq->cqn_handler = NULL;
 437         nq->srqn_handler = NULL;
 438         nq->msix_vec = 0;
 439 }
 440
 441 int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
 442                             int msix_vector, bool need_init)
 443 {
 444         int rc;
 445
 446         if (nq->requested)
 447                 return -EFAULT;
 448
 449         nq->msix_vec = msix_vector;
 450         if (need_init)
 451                 tasklet_init(&nq->nq_tasklet, bnxt_qplib_service_nq,
 452                              (unsigned long)nq);
 453         else
 454                 tasklet_enable(&nq->nq_tasklet);
 455
 456         snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx);
 457         rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
 458         if (rc)
 459                 return rc;
 460
 461         cpumask_clear(&nq->mask);
 462         cpumask_set_cpu(nq_indx, &nq->mask);
 463         rc = irq_set_affinity_hint(nq->msix_vec, &nq->mask);
 464         if (rc) {
 465                 dev_warn(&nq->pdev->dev,
 466                          "set affinity failed; vector: %d nq_idx: %d\n",
 467                          nq->msix_vec, nq_indx);
 468         }
 469         nq->requested = true;
 470         bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
 471
 472         return rc;
 473 }
 474
 475 static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq,  u32 reg_offt)
 476 {
 477         resource_size_t reg_base;
 478         struct bnxt_qplib_nq_db *nq_db;
 479         struct pci_dev *pdev;
 480         int rc = 0;
 481
 482         pdev = nq->pdev;
 483         nq_db = &nq->nq_db;
 484
 485         nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION;
 486         nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id);
 487         if (!nq_db->reg.bar_base) {
 488                 dev_err(&pdev->dev, "QPLIB: NQ BAR region %d resc start is 0!",
 489                         nq_db->reg.bar_id);
 490                 rc = -ENOMEM;
 491                 goto fail;
 492         }
 493
 494         reg_base = nq_db->reg.bar_base + reg_offt;
 495         /* Unconditionally map 8 bytes to support 57500 series */
 496         nq_db->reg.len = 8;
 497         nq_db->reg.bar_reg = ioremap(reg_base, nq_db->reg.len);
 498         if (!nq_db->reg.bar_reg) {
 499                 dev_err(&pdev->dev, "QPLIB: NQ BAR region %d mapping failed",
 500                         nq_db->reg.bar_id);
 501                 rc = -ENOMEM;
 502                 goto fail;
 503         }
 504
 505         nq_db->dbinfo.db = nq_db->reg.bar_reg;
 506         nq_db->dbinfo.hwq = &nq->hwq;
 507         nq_db->dbinfo.xid = nq->ring_id;
 508 fail:
 509         return rc;
 510 }
 511
 512 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 513                          int nq_idx, int msix_vector, int bar_reg_offset,
 514                          cqn_handler_t cqn_handler,
 515                          srqn_handler_t srqn_handler)
 516 {
 517         int rc = -1;
 518
 519         nq->pdev = pdev;
 520         nq->cqn_handler = cqn_handler;
 521         nq->srqn_handler = srqn_handler;
 522
 523         /* Have a task to schedule CQ notifiers in post send case */
 524         nq->cqn_wq  = create_singlethread_workqueue("bnxt_qplib_nq");
 525         if (!nq->cqn_wq)
 526                 return -ENOMEM;
 527
 528         rc = bnxt_qplib_map_nq_db(nq, bar_reg_offset);
 529         if (rc)
 530                 goto fail;
 531
 532         rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
 533         if (rc) {
 534                 dev_err(&nq->pdev->dev,
 535                         "Failed to request irq for nq-idx %d\n", nq_idx);
 536                 goto fail;
 537         }
 538
 539         return 0;
 540 fail:
 541         bnxt_qplib_disable_nq(nq);
 542         return rc;
 543 }
 544
 545 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
 546 {
 547         if (nq->hwq.max_elements) {
 548                 bnxt_qplib_free_hwq(nq->res, &nq->hwq);
 549                 nq->hwq.max_elements = 0;
 550         }
 551 }
 552
 553 int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq)
 554 {
 555         struct bnxt_qplib_hwq_attr hwq_attr = {};
 556         struct bnxt_qplib_sg_info sginfo = {};
 557
 558         nq->pdev = res->pdev;
 559         nq->res = res;
 560         if (!nq->hwq.max_elements ||
 561             nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
 562                 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
 563
 564         sginfo.pgsize = PAGE_SIZE;
 565         sginfo.pgshft = PAGE_SHIFT;
 566         hwq_attr.res = res;
 567         hwq_attr.sginfo = &sginfo;
 568         hwq_attr.depth = nq->hwq.max_elements;
 569         hwq_attr.stride = sizeof(struct nq_base);
 570         hwq_attr.type = bnxt_qplib_get_hwq_type(nq->res);
 571         if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
 572                 dev_err(&nq->pdev->dev, "FP NQ allocation failed");
 573                 return -ENOMEM;
 574         }
 575         nq->budget = 8;
 576         return 0;
 577 }
 578
 579 /* SRQ */
 580 void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
 581                            struct bnxt_qplib_srq *srq)
 582 {
 583         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 584         struct cmdq_destroy_srq req;
 585         struct creq_destroy_srq_resp resp;
 586         u16 cmd_flags = 0;
 587         int rc;
 588
 589         RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);
 590
 591         /* Configure the request */
 592         req.srq_cid = cpu_to_le32(srq->id);
 593
 594         rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&req,
 595                                           (struct creq_base *)&resp, NULL, 0);
 596         kfree(srq->swq);
 597         if (rc)
 598                 return;
 599         bnxt_qplib_free_hwq(res, &srq->hwq);
 600 }
 601
 602 int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
 603                           struct bnxt_qplib_srq *srq)
 604 {
 605         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 606         struct bnxt_qplib_hwq_attr hwq_attr = {};
 607         struct creq_create_srq_resp resp;
 608         struct cmdq_create_srq req;
 609         struct bnxt_qplib_pbl *pbl;
 610         u16 cmd_flags = 0;
 611         u16 pg_sz_lvl;
 612         int rc, idx;
 613
 614         hwq_attr.res = res;
 615         hwq_attr.sginfo = &srq->sg_info;
 616         hwq_attr.depth = srq->max_wqe;
 617         hwq_attr.stride = srq->wqe_size;
 618         hwq_attr.type = HWQ_TYPE_QUEUE;
 619         rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
 620         if (rc)
 621                 goto exit;
 622
 623         srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),
 624                            GFP_KERNEL);
 625         if (!srq->swq) {
 626                 rc = -ENOMEM;
 627                 goto fail;
 628         }
 629
 630         RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);
 631
 632         /* Configure the request */
 633         req.dpi = cpu_to_le32(srq->dpi->dpi);
 634         req.srq_handle = cpu_to_le64((uintptr_t)srq);
 635
 636         req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
 637         pbl = &srq->hwq.pbl[PBL_LVL_0];
 638         pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) <<
 639                      CMDQ_CREATE_SRQ_PG_SIZE_SFT);
 640         pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) <<
 641                       CMDQ_CREATE_SRQ_LVL_SFT;
 642         req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
 643         req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 644         req.pd_id = cpu_to_le32(srq->pd->id);
 645         req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
 646
 647         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 648                                           (void *)&resp, NULL, 0);
 649         if (rc)
 650                 goto fail;
 651
 652         spin_lock_init(&srq->lock);
 653         srq->start_idx = 0;
 654         srq->last_idx = srq->hwq.max_elements - 1;
 655         for (idx = 0; idx < srq->hwq.max_elements; idx++)
 656                 srq->swq[idx].next_idx = idx + 1;
 657         srq->swq[srq->last_idx].next_idx = -1;
 658
 659         srq->id = le32_to_cpu(resp.xid);
 660         srq->dbinfo.hwq = &srq->hwq;
 661         srq->dbinfo.xid = srq->id;
 662         srq->dbinfo.db = srq->dpi->dbr;
 663         srq->dbinfo.max_slot = 1;
 664         srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
 665         if (srq->threshold)
 666                 bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
 667         srq->arm_req = false;
 668
 669         return 0;
 670 fail:
 671         bnxt_qplib_free_hwq(res, &srq->hwq);
 672         kfree(srq->swq);
 673 exit:
 674         return rc;
 675 }
 676
 677 int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
 678                           struct bnxt_qplib_srq *srq)
 679 {
 680         struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
 681         u32 sw_prod, sw_cons, count = 0;
 682
 683         sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
 684         sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
 685
 686         count = sw_prod > sw_cons ? sw_prod - sw_cons :
 687                                     srq_hwq->max_elements - sw_cons + sw_prod;
 688         if (count > srq->threshold) {
 689                 srq->arm_req = false;
 690                 bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
 691         } else {
 692                 /* Deferred arming */
 693                 srq->arm_req = true;
 694         }
 695
 696         return 0;
 697 }
 698
 699 int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
 700                          struct bnxt_qplib_srq *srq)
 701 {
 702         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 703         struct cmdq_query_srq req;
 704         struct creq_query_srq_resp resp;
 705         struct bnxt_qplib_rcfw_sbuf *sbuf;
 706         struct creq_query_srq_resp_sb *sb;
 707         u16 cmd_flags = 0;
 708         int rc = 0;
 709
 710         RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);
 711         req.srq_cid = cpu_to_le32(srq->id);
 712
 713         /* Configure the request */
 714         sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
 715         if (!sbuf)
 716                 return -ENOMEM;
 717         sb = sbuf->sb;
 718         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
 719                                           (void *)sbuf, 0);
 720         srq->threshold = le16_to_cpu(sb->srq_limit);
 721         bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
 722
 723         return rc;
 724 }
 725
 726 int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 727                              struct bnxt_qplib_swqe *wqe)
 728 {
 729         struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
 730         struct rq_wqe *srqe;
 731         struct sq_sge *hw_sge;
 732         u32 sw_prod, sw_cons, count = 0;
 733         int i, rc = 0, next;
 734
 735         spin_lock(&srq_hwq->lock);
 736         if (srq->start_idx == srq->last_idx) {
 737                 dev_err(&srq_hwq->pdev->dev,
 738                         "FP: SRQ (0x%x) is full!\n", srq->id);
 739                 rc = -EINVAL;
 740                 spin_unlock(&srq_hwq->lock);
 741                 goto done;
 742         }
 743         next = srq->start_idx;
 744         srq->start_idx = srq->swq[next].next_idx;
 745         spin_unlock(&srq_hwq->lock);
 746
 747         sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
 748         srqe = bnxt_qplib_get_qe(srq_hwq, sw_prod, NULL);
 749         memset(srqe, 0, srq->wqe_size);
 750         /* Calculate wqe_size16 and data_len */
 751         for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
 752              i < wqe->num_sge; i++, hw_sge++) {
 753                 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
 754                 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
 755                 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
 756         }
 757         srqe->wqe_type = wqe->type;
 758         srqe->flags = wqe->flags;
 759         srqe->wqe_size = wqe->num_sge +
 760                         ((offsetof(typeof(*srqe), data) + 15) >> 4);
 761         srqe->wr_id[0] = cpu_to_le32((u32)next);
 762         srq->swq[next].wr_id = wqe->wr_id;
 763
 764         srq_hwq->prod++;
 765
 766         spin_lock(&srq_hwq->lock);
 767         sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
 768         /* retaining srq_hwq->cons for this logic
 769          * actually the lock is only required to
 770          * read srq_hwq->cons.
 771          */
 772         sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
 773         count = sw_prod > sw_cons ? sw_prod - sw_cons :
 774                                     srq_hwq->max_elements - sw_cons + sw_prod;
 775         spin_unlock(&srq_hwq->lock);
 776         /* Ring DB */
 777         bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
 778         if (srq->arm_req == true && count > srq->threshold) {
 779                 srq->arm_req = false;
 780                 bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
 781         }
 782 done:
 783         return rc;
 784 }
 785
 786 /* QP */
 787
 788 static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
 789 {
 790         int rc = 0;
 791         int indx;
 792
 793         que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
 794         if (!que->swq) {
 795                 rc = -ENOMEM;
 796                 goto out;
 797         }
 798
 799         que->swq_start = 0;
 800         que->swq_last = que->max_wqe - 1;
 801         for (indx = 0; indx < que->max_wqe; indx++)
 802                 que->swq[indx].next_idx = indx + 1;
 803         que->swq[que->swq_last].next_idx = 0; /* Make it circular */
 804         que->swq_last = 0;
 805 out:
 806         return rc;
 807 }
 808
 809 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 810 {
 811         struct bnxt_qplib_hwq_attr hwq_attr = {};
 812         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 813         struct bnxt_qplib_q *sq = &qp->sq;
 814         struct bnxt_qplib_q *rq = &qp->rq;
 815         struct creq_create_qp1_resp resp;
 816         struct cmdq_create_qp1 req;
 817         struct bnxt_qplib_pbl *pbl;
 818         u16 cmd_flags = 0;
 819         u32 qp_flags = 0;
 820         u8 pg_sz_lvl;
 821         int rc;
 822
 823         RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
 824
 825         /* General */
 826         req.type = qp->type;
 827         req.dpi = cpu_to_le32(qp->dpi->dpi);
 828         req.qp_handle = cpu_to_le64(qp->qp_handle);
 829
 830         /* SQ */
 831         hwq_attr.res = res;
 832         hwq_attr.sginfo = &sq->sg_info;
 833         hwq_attr.stride = sizeof(struct sq_sge);
 834         hwq_attr.depth = bnxt_qplib_get_depth(sq);
 835         hwq_attr.type = HWQ_TYPE_QUEUE;
 836         rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
 837         if (rc)
 838                 goto exit;
 839
 840         rc = bnxt_qplib_alloc_init_swq(sq);
 841         if (rc)
 842                 goto fail_sq;
 843
 844         req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
 845         pbl = &sq->hwq.pbl[PBL_LVL_0];
 846         req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 847         pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
 848                      CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
 849         pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
 850         req.sq_pg_size_sq_lvl = pg_sz_lvl;
 851         req.sq_fwo_sq_sge =
 852                 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
 853                              CMDQ_CREATE_QP1_SQ_SGE_SFT);
 854         req.scq_cid = cpu_to_le32(qp->scq->id);
 855
 856         /* RQ */
 857         if (rq->max_wqe) {
 858                 hwq_attr.res = res;
 859                 hwq_attr.sginfo = &rq->sg_info;
 860                 hwq_attr.stride = sizeof(struct sq_sge);
 861                 hwq_attr.depth = bnxt_qplib_get_depth(rq);
 862                 hwq_attr.type = HWQ_TYPE_QUEUE;
 863                 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
 864                 if (rc)
 865                         goto sq_swq;
 866                 rc = bnxt_qplib_alloc_init_swq(rq);
 867                 if (rc)
 868                         goto fail_rq;
 869                 req.rq_size = cpu_to_le32(rq->max_wqe);
 870                 pbl = &rq->hwq.pbl[PBL_LVL_0];
 871                 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 872                 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
 873                              CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
 874                 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
 875                 req.rq_pg_size_rq_lvl = pg_sz_lvl;
 876                 req.rq_fwo_rq_sge =
 877                         cpu_to_le16((rq->max_sge &
 878                                      CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
 879                                     CMDQ_CREATE_QP1_RQ_SGE_SFT);
 880         }
 881         req.rcq_cid = cpu_to_le32(qp->rcq->id);
 882         /* Header buffer - allow hdr_buf pass in */
 883         rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 884         if (rc) {
 885                 rc = -ENOMEM;
 886                 goto rq_rwq;
 887         }
 888         qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
 889         req.qp_flags = cpu_to_le32(qp_flags);
 890         req.pd_id = cpu_to_le32(qp->pd->id);
 891
 892         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 893                                           (void *)&resp, NULL, 0);
 894         if (rc)
 895                 goto fail;
 896
 897         qp->id = le32_to_cpu(resp.xid);
 898         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
 899         qp->cctx = res->cctx;
 900         sq->dbinfo.hwq = &sq->hwq;
 901         sq->dbinfo.xid = qp->id;
 902         sq->dbinfo.db = qp->dpi->dbr;
 903         sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
 904         if (rq->max_wqe) {
 905                 rq->dbinfo.hwq = &rq->hwq;
 906                 rq->dbinfo.xid = qp->id;
 907                 rq->dbinfo.db = qp->dpi->dbr;
 908                 rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
 909         }
 910         rcfw->qp_tbl[qp->id].qp_id = qp->id;
 911         rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
 912
 913         return 0;
 914
 915 fail:
 916         bnxt_qplib_free_qp_hdr_buf(res, qp);
 917 rq_rwq:
 918         kfree(rq->swq);
 919 fail_rq:
 920         bnxt_qplib_free_hwq(res, &rq->hwq);
 921 sq_swq:
 922         kfree(sq->swq);
 923 fail_sq:
 924         bnxt_qplib_free_hwq(res, &sq->hwq);
 925 exit:
 926         return rc;
 927 }
 928
 929 static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
 930 {
 931         struct bnxt_qplib_hwq *hwq;
 932         struct bnxt_qplib_q *sq;
 933         u64 fpsne, psn_pg;
 934         u16 indx_pad = 0;
 935
 936         sq = &qp->sq;
 937         hwq = &sq->hwq;
 938         fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg);
 939         if (!IS_ALIGNED(fpsne, PAGE_SIZE))
 940                 indx_pad = ALIGN(fpsne, PAGE_SIZE) / size;
 941
 942         hwq->pad_pgofft = indx_pad;
 943         hwq->pad_pg = (u64 *)psn_pg;
 944         hwq->pad_stride = size;
 945 }
 946
 947 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 948 {
 949         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 950         struct bnxt_qplib_hwq_attr hwq_attr = {};
 951         struct bnxt_qplib_sg_info sginfo = {};
 952         struct bnxt_qplib_q *sq = &qp->sq;
 953         struct bnxt_qplib_q *rq = &qp->rq;
 954         struct creq_create_qp_resp resp;
 955         int rc, req_size, psn_sz = 0;
 956         struct bnxt_qplib_hwq *xrrq;
 957         struct bnxt_qplib_pbl *pbl;
 958         struct cmdq_create_qp req;
 959         u16 cmd_flags = 0;
 960         u32 qp_flags = 0;
 961         u8 pg_sz_lvl;
 962         u16 nsge;
 963
 964         RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
 965
 966         /* General */
 967         req.type = qp->type;
 968         req.dpi = cpu_to_le32(qp->dpi->dpi);
 969         req.qp_handle = cpu_to_le64(qp->qp_handle);
 970
 971         /* SQ */
 972         if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
 973                 psn_sz = bnxt_qplib_is_chip_gen_p5(res->cctx) ?
 974                          sizeof(struct sq_psn_search_ext) :
 975                          sizeof(struct sq_psn_search);
 976         }
 977
 978         hwq_attr.res = res;
 979         hwq_attr.sginfo = &sq->sg_info;
 980         hwq_attr.stride = sizeof(struct sq_sge);
 981         hwq_attr.depth = bnxt_qplib_get_depth(sq);
 982         hwq_attr.aux_stride = psn_sz;
 983         hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode);
 984         hwq_attr.type = HWQ_TYPE_QUEUE;
 985         rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
 986         if (rc)
 987                 goto exit;
 988
 989         rc = bnxt_qplib_alloc_init_swq(sq);
 990         if (rc)
 991                 goto fail_sq;
 992
 993         if (psn_sz)
 994                 bnxt_qplib_init_psn_ptr(qp, psn_sz);
 995
 996         req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
 997         pbl = &sq->hwq.pbl[PBL_LVL_0];
 998         req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 999         pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
1000                      CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
1001         pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
1002         req.sq_pg_size_sq_lvl = pg_sz_lvl;
1003         req.sq_fwo_sq_sge =
1004                 cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
1005                              CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
1006         req.scq_cid = cpu_to_le32(qp->scq->id);
1007
1008         /* RQ */
1009         if (!qp->srq) {
1010                 hwq_attr.res = res;
1011                 hwq_attr.sginfo = &rq->sg_info;
1012                 hwq_attr.stride = sizeof(struct sq_sge);
1013                 hwq_attr.depth = bnxt_qplib_get_depth(rq);
1014                 hwq_attr.aux_stride = 0;
1015                 hwq_attr.aux_depth = 0;
1016                 hwq_attr.type = HWQ_TYPE_QUEUE;
1017                 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
1018                 if (rc)
1019                         goto sq_swq;
1020                 rc = bnxt_qplib_alloc_init_swq(rq);
1021                 if (rc)
1022                         goto fail_rq;
1023
1024                 req.rq_size = cpu_to_le32(rq->max_wqe);
1025                 pbl = &rq->hwq.pbl[PBL_LVL_0];
1026                 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1027                 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
1028                              CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
1029                 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
1030                 req.rq_pg_size_rq_lvl = pg_sz_lvl;
1031                 nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1032                         6 : rq->max_sge;
1033                 req.rq_fwo_rq_sge =
1034                         cpu_to_le16(((nsge &
1035                                       CMDQ_CREATE_QP_RQ_SGE_MASK) <<
1036                                      CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
1037         } else {
1038                 /* SRQ */
1039                 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
1040                 req.srq_cid = cpu_to_le32(qp->srq->id);
1041         }
1042         req.rcq_cid = cpu_to_le32(qp->rcq->id);
1043
1044         qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
1045         qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
1046         if (qp->sig_type)
1047                 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
1048         if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
1049                 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
1050         req.qp_flags = cpu_to_le32(qp_flags);
1051
1052         /* ORRQ and IRRQ */
1053         if (psn_sz) {
1054                 xrrq = &qp->orrq;
1055                 xrrq->max_elements =
1056                         ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1057                 req_size = xrrq->max_elements *
1058                            BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1059                 req_size &= ~(PAGE_SIZE - 1);
1060                 sginfo.pgsize = req_size;
1061                 sginfo.pgshft = PAGE_SHIFT;
1062
1063                 hwq_attr.res = res;
1064                 hwq_attr.sginfo = &sginfo;
1065                 hwq_attr.depth = xrrq->max_elements;
1066                 hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
1067                 hwq_attr.aux_stride = 0;
1068                 hwq_attr.aux_depth = 0;
1069                 hwq_attr.type = HWQ_TYPE_CTX;
1070                 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1071                 if (rc)
1072                         goto rq_swq;
1073                 pbl = &xrrq->pbl[PBL_LVL_0];
1074                 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1075
1076                 xrrq = &qp->irrq;
1077                 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
1078                                                 qp->max_dest_rd_atomic);
1079                 req_size = xrrq->max_elements *
1080                            BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1081                 req_size &= ~(PAGE_SIZE - 1);
1082                 sginfo.pgsize = req_size;
1083                 hwq_attr.depth =  xrrq->max_elements;
1084                 hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
1085                 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1086                 if (rc)
1087                         goto fail_orrq;
1088
1089                 pbl = &xrrq->pbl[PBL_LVL_0];
1090                 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1091         }
1092         req.pd_id = cpu_to_le32(qp->pd->id);
1093
1094         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1095                                           (void *)&resp, NULL, 0);
1096         if (rc)
1097                 goto fail;
1098
1099         qp->id = le32_to_cpu(resp.xid);
1100         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
1101         INIT_LIST_HEAD(&qp->sq_flush);
1102         INIT_LIST_HEAD(&qp->rq_flush);
1103         qp->cctx = res->cctx;
1104         sq->dbinfo.hwq = &sq->hwq;
1105         sq->dbinfo.xid = qp->id;
1106         sq->dbinfo.db = qp->dpi->dbr;
1107         sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
1108         if (rq->max_wqe) {
1109                 rq->dbinfo.hwq = &rq->hwq;
1110                 rq->dbinfo.xid = qp->id;
1111                 rq->dbinfo.db = qp->dpi->dbr;
1112                 rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
1113         }
1114         rcfw->qp_tbl[qp->id].qp_id = qp->id;
1115         rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
1116
1117         return 0;
1118 fail:
1119         bnxt_qplib_free_hwq(res, &qp->irrq);
1120 fail_orrq:
1121         bnxt_qplib_free_hwq(res, &qp->orrq);
1122 rq_swq:
1123         kfree(rq->swq);
1124 fail_rq:
1125         bnxt_qplib_free_hwq(res, &rq->hwq);
1126 sq_swq:
1127         kfree(sq->swq);
1128 fail_sq:
1129         bnxt_qplib_free_hwq(res, &sq->hwq);
1130 exit:
1131         return rc;
1132 }
1133
1134 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
1135 {
1136         switch (qp->state) {
1137         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1138                 /* INIT->RTR, configure the path_mtu to the default
1139                  * 2048 if not being requested
1140                  */
1141                 if (!(qp->modify_flags &
1142                     CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
1143                         qp->modify_flags |=
1144                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1145                         qp->path_mtu =
1146                                 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1147                 }
1148                 qp->modify_flags &=
1149                         ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1150                 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
1151                 if (qp->max_dest_rd_atomic < 1)
1152                         qp->max_dest_rd_atomic = 1;
1153                 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
1154                 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
1155                 if (!(qp->modify_flags &
1156                     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
1157                         qp->modify_flags |=
1158                                 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
1159                         qp->ah.sgid_index = 0;
1160                 }
1161                 break;
1162         default:
1163                 break;
1164         }
1165 }
1166
1167 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
1168 {
1169         switch (qp->state) {
1170         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1171                 /* Bono FW requires the max_rd_atomic to be >= 1 */
1172                 if (qp->max_rd_atomic < 1)
1173                         qp->max_rd_atomic = 1;
1174                 /* Bono FW does not allow PKEY_INDEX,
1175                  * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
1176                  * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
1177                  * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
1178                  * modification
1179                  */
1180                 qp->modify_flags &=
1181                         ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
1182                           CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1183                           CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1184                           CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1185                           CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1186                           CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1187                           CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1188                           CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
1189                           CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
1190                           CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
1191                           CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
1192                           CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
1193                 break;
1194         default:
1195                 break;
1196         }
1197 }
1198
1199 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
1200 {
1201         switch (qp->cur_qp_state) {
1202         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1203                 break;
1204         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1205                 __modify_flags_from_init_state(qp);
1206                 break;
1207         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1208                 __modify_flags_from_rtr_state(qp);
1209                 break;
1210         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1211                 break;
1212         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1213                 break;
1214         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1215                 break;
1216         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1217                 break;
1218         default:
1219                 break;
1220         }
1221 }
1222
1223 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1224 {
1225         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1226         struct cmdq_modify_qp req;
1227         struct creq_modify_qp_resp resp;
1228         u16 cmd_flags = 0, pkey;
1229         u32 temp32[4];
1230         u32 bmask;
1231         int rc;
1232
1233         RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
1234
1235         /* Filter out the qp_attr_mask based on the state->new transition */
1236         __filter_modify_flags(qp);
1237         bmask = qp->modify_flags;
1238         req.modify_mask = cpu_to_le32(qp->modify_flags);
1239         req.qp_cid = cpu_to_le32(qp->id);
1240         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
1241                 req.network_type_en_sqd_async_notify_new_state =
1242                                 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
1243                                 (qp->en_sqd_async_notify ?
1244                                         CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
1245         }
1246         req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
1247
1248         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
1249                 req.access = qp->access;
1250
1251         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
1252                 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
1253                                          qp->pkey_index, &pkey))
1254                         req.pkey = cpu_to_le16(pkey);
1255         }
1256         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
1257                 req.qkey = cpu_to_le32(qp->qkey);
1258
1259         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
1260                 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
1261                 req.dgid[0] = cpu_to_le32(temp32[0]);
1262                 req.dgid[1] = cpu_to_le32(temp32[1]);
1263                 req.dgid[2] = cpu_to_le32(temp32[2]);
1264                 req.dgid[3] = cpu_to_le32(temp32[3]);
1265         }
1266         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
1267                 req.flow_label = cpu_to_le32(qp->ah.flow_label);
1268
1269         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
1270                 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
1271                                              [qp->ah.sgid_index]);
1272
1273         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
1274                 req.hop_limit = qp->ah.hop_limit;
1275
1276         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
1277                 req.traffic_class = qp->ah.traffic_class;
1278
1279         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
1280                 memcpy(req.dest_mac, qp->ah.dmac, 6);
1281
1282         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
1283                 req.path_mtu = qp->path_mtu;
1284
1285         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
1286                 req.timeout = qp->timeout;
1287
1288         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
1289                 req.retry_cnt = qp->retry_cnt;
1290
1291         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
1292                 req.rnr_retry = qp->rnr_retry;
1293
1294         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
1295                 req.min_rnr_timer = qp->min_rnr_timer;
1296
1297         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
1298                 req.rq_psn = cpu_to_le32(qp->rq.psn);
1299
1300         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
1301                 req.sq_psn = cpu_to_le32(qp->sq.psn);
1302
1303         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
1304                 req.max_rd_atomic =
1305                         ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1306
1307         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
1308                 req.max_dest_rd_atomic =
1309                         IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
1310
1311         req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
1312         req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
1313         req.sq_sge = cpu_to_le16(qp->sq.max_sge);
1314         req.rq_sge = cpu_to_le16(qp->rq.max_sge);
1315         req.max_inline_data = cpu_to_le32(qp->max_inline_data);
1316         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
1317                 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
1318
1319         req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
1320
1321         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1322                                           (void *)&resp, NULL, 0);
1323         if (rc)
1324                 return rc;
1325         qp->cur_qp_state = qp->state;
1326         return 0;
1327 }
1328
1329 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1330 {
1331         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1332         struct cmdq_query_qp req;
1333         struct creq_query_qp_resp resp;
1334         struct bnxt_qplib_rcfw_sbuf *sbuf;
1335         struct creq_query_qp_resp_sb *sb;
1336         u16 cmd_flags = 0;
1337         u32 temp32[4];
1338         int i, rc = 0;
1339
1340         RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
1341
1342         sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
1343         if (!sbuf)
1344                 return -ENOMEM;
1345         sb = sbuf->sb;
1346
1347         req.qp_cid = cpu_to_le32(qp->id);
1348         req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
1349         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
1350                                           (void *)sbuf, 0);
1351         if (rc)
1352                 goto bail;
1353         /* Extract the context from the side buffer */
1354         qp->state = sb->en_sqd_async_notify_state &
1355                         CREQ_QUERY_QP_RESP_SB_STATE_MASK;
1356         qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
1357                                   CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
1358                                   true : false;
1359         qp->access = sb->access;
1360         qp->pkey_index = le16_to_cpu(sb->pkey);
1361         qp->qkey = le32_to_cpu(sb->qkey);
1362
1363         temp32[0] = le32_to_cpu(sb->dgid[0]);
1364         temp32[1] = le32_to_cpu(sb->dgid[1]);
1365         temp32[2] = le32_to_cpu(sb->dgid[2]);
1366         temp32[3] = le32_to_cpu(sb->dgid[3]);
1367         memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
1368
1369         qp->ah.flow_label = le32_to_cpu(sb->flow_label);
1370
1371         qp->ah.sgid_index = 0;
1372         for (i = 0; i < res->sgid_tbl.max; i++) {
1373                 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
1374                         qp->ah.sgid_index = i;
1375                         break;
1376                 }
1377         }
1378         if (i == res->sgid_tbl.max)
1379                 dev_warn(&res->pdev->dev, "SGID not found??\n");
1380
1381         qp->ah.hop_limit = sb->hop_limit;
1382         qp->ah.traffic_class = sb->traffic_class;
1383         memcpy(qp->ah.dmac, sb->dest_mac, 6);
1384         qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1385                                 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
1386                                 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
1387         qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1388                                     CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
1389                                     CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
1390         qp->timeout = sb->timeout;
1391         qp->retry_cnt = sb->retry_cnt;
1392         qp->rnr_retry = sb->rnr_retry;
1393         qp->min_rnr_timer = sb->min_rnr_timer;
1394         qp->rq.psn = le32_to_cpu(sb->rq_psn);
1395         qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
1396         qp->sq.psn = le32_to_cpu(sb->sq_psn);
1397         qp->max_dest_rd_atomic =
1398                         IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
1399         qp->sq.max_wqe = qp->sq.hwq.max_elements;
1400         qp->rq.max_wqe = qp->rq.hwq.max_elements;
1401         qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
1402         qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
1403         qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
1404         qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
1405         memcpy(qp->smac, sb->src_mac, 6);
1406         qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
1407 bail:
1408         bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
1409         return rc;
1410 }
1411
1412 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1413 {
1414         struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1415         struct cq_base *hw_cqe;
1416         int i;
1417
1418         for (i = 0; i < cq_hwq->max_elements; i++) {
1419                 hw_cqe = bnxt_qplib_get_qe(cq_hwq, i, NULL);
1420                 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
1421                         continue;
1422                 /*
1423                  * The valid test of the entry must be done first before
1424                  * reading any further.
1425                  */
1426                 dma_rmb();
1427                 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
1428                 case CQ_BASE_CQE_TYPE_REQ:
1429                 case CQ_BASE_CQE_TYPE_TERMINAL:
1430                 {
1431                         struct cq_req *cqe = (struct cq_req *)hw_cqe;
1432
1433                         if (qp == le64_to_cpu(cqe->qp_handle))
1434                                 cqe->qp_handle = 0;
1435                         break;
1436                 }
1437                 case CQ_BASE_CQE_TYPE_RES_RC:
1438                 case CQ_BASE_CQE_TYPE_RES_UD:
1439                 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1440                 {
1441                         struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1442
1443                         if (qp == le64_to_cpu(cqe->qp_handle))
1444                                 cqe->qp_handle = 0;
1445                         break;
1446                 }
1447                 default:
1448                         break;
1449                 }
1450         }
1451 }
1452
1453 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1454                           struct bnxt_qplib_qp *qp)
1455 {
1456         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1457         struct cmdq_destroy_qp req;
1458         struct creq_destroy_qp_resp resp;
1459         u16 cmd_flags = 0;
1460         int rc;
1461
1462         rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
1463         rcfw->qp_tbl[qp->id].qp_handle = NULL;
1464
1465         RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1466
1467         req.qp_cid = cpu_to_le32(qp->id);
1468         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1469                                           (void *)&resp, NULL, 0);
1470         if (rc) {
1471                 rcfw->qp_tbl[qp->id].qp_id = qp->id;
1472                 rcfw->qp_tbl[qp->id].qp_handle = qp;
1473                 return rc;
1474         }
1475
1476         return 0;
1477 }
1478
1479 void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
1480                             struct bnxt_qplib_qp *qp)
1481 {
1482         bnxt_qplib_free_qp_hdr_buf(res, qp);
1483         bnxt_qplib_free_hwq(res, &qp->sq.hwq);
1484         kfree(qp->sq.swq);
1485
1486         bnxt_qplib_free_hwq(res, &qp->rq.hwq);
1487         kfree(qp->rq.swq);
1488
1489         if (qp->irrq.max_elements)
1490                 bnxt_qplib_free_hwq(res, &qp->irrq);
1491         if (qp->orrq.max_elements)
1492                 bnxt_qplib_free_hwq(res, &qp->orrq);
1493
1494 }
1495
1496 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1497                                 struct bnxt_qplib_sge *sge)
1498 {
1499         struct bnxt_qplib_q *sq = &qp->sq;
1500         u32 sw_prod;
1501
1502         memset(sge, 0, sizeof(*sge));
1503
1504         if (qp->sq_hdr_buf) {
1505                 sw_prod = sq->swq_start;
1506                 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1507                                          sw_prod * qp->sq_hdr_buf_size);
1508                 sge->lkey = 0xFFFFFFFF;
1509                 sge->size = qp->sq_hdr_buf_size;
1510                 return qp->sq_hdr_buf + sw_prod * sge->size;
1511         }
1512         return NULL;
1513 }
1514
1515 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1516 {
1517         struct bnxt_qplib_q *rq = &qp->rq;
1518
1519         return rq->swq_start;
1520 }
1521
1522 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1523 {
1524         return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1525 }
1526
1527 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1528                                 struct bnxt_qplib_sge *sge)
1529 {
1530         struct bnxt_qplib_q *rq = &qp->rq;
1531         u32 sw_prod;
1532
1533         memset(sge, 0, sizeof(*sge));
1534
1535         if (qp->rq_hdr_buf) {
1536                 sw_prod = rq->swq_start;
1537                 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1538                                          sw_prod * qp->rq_hdr_buf_size);
1539                 sge->lkey = 0xFFFFFFFF;
1540                 sge->size = qp->rq_hdr_buf_size;
1541                 return qp->rq_hdr_buf + sw_prod * sge->size;
1542         }
1543         return NULL;
1544 }
1545
1546 static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
1547                                        struct bnxt_qplib_swqe *wqe,
1548                                        struct bnxt_qplib_swq *swq)
1549 {
1550         struct sq_psn_search_ext *psns_ext;
1551         struct sq_psn_search *psns;
1552         u32 flg_npsn;
1553         u32 op_spsn;
1554
1555         if (!swq->psn_search)
1556                 return;
1557         psns = swq->psn_search;
1558         psns_ext = swq->psn_ext;
1559
1560         op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1561                     SQ_PSN_SEARCH_START_PSN_MASK);
1562         op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1563                      SQ_PSN_SEARCH_OPCODE_MASK);
1564         flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1565                      SQ_PSN_SEARCH_NEXT_PSN_MASK);
1566
1567         if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
1568                 psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
1569                 psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
1570                 psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
1571         } else {
1572                 psns->opcode_start_psn = cpu_to_le32(op_spsn);
1573                 psns->flags_next_psn = cpu_to_le32(flg_npsn);
1574         }
1575 }
1576
1577 static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
1578                                  struct bnxt_qplib_swqe *wqe,
1579                                  u16 *idx)
1580 {
1581         struct bnxt_qplib_hwq *hwq;
1582         int len, t_len, offt;
1583         bool pull_dst = true;
1584         void *il_dst = NULL;
1585         void *il_src = NULL;
1586         int t_cplen, cplen;
1587         int indx;
1588
1589         hwq = &qp->sq.hwq;
1590         t_len = 0;
1591         for (indx = 0; indx < wqe->num_sge; indx++) {
1592                 len = wqe->sg_list[indx].size;
1593                 il_src = (void *)wqe->sg_list[indx].addr;
1594                 t_len += len;
1595                 if (t_len > qp->max_inline_data)
1596                         goto bad;
1597                 while (len) {
1598                         if (pull_dst) {
1599                                 pull_dst = false;
1600                                 il_dst = bnxt_qplib_get_prod_qe(hwq, *idx);
1601                                 (*idx)++;
1602                                 t_cplen = 0;
1603                                 offt = 0;
1604                         }
1605                         cplen = min_t(int, len, sizeof(struct sq_sge));
1606                         cplen = min_t(int, cplen,
1607                                         (sizeof(struct sq_sge) - offt));
1608                         memcpy(il_dst, il_src, cplen);
1609                         t_cplen += cplen;
1610                         il_src += cplen;
1611                         il_dst += cplen;
1612                         offt += cplen;
1613                         len -= cplen;
1614                         if (t_cplen == sizeof(struct sq_sge))
1615                                 pull_dst = true;
1616                 }
1617         }
1618
1619         return t_len;
1620 bad:
1621         return -ENOMEM;
1622 }
1623
1624 static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
1625                                struct bnxt_qplib_sge *ssge,
1626                                u16 nsge, u16 *idx)
1627 {
1628         struct sq_sge *dsge;
1629         int indx, len = 0;
1630
1631         for (indx = 0; indx < nsge; indx++, (*idx)++) {
1632                 dsge = bnxt_qplib_get_prod_qe(hwq, *idx);
1633                 dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
1634                 dsge->l_key = cpu_to_le32(ssge[indx].lkey);
1635                 dsge->size = cpu_to_le32(ssge[indx].size);
1636                 len += ssge[indx].size;
1637         }
1638
1639         return len;
1640 }
1641
1642 static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp,
1643                                      struct bnxt_qplib_swqe *wqe,
1644                                      u16 *wqe_sz, u16 *qdf, u8 mode)
1645 {
1646         u32 ilsize, bytes;
1647         u16 nsge;
1648         u16 slot;
1649
1650         nsge = wqe->num_sge;
1651         /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
1652         bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
1653         if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1654                 ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data);
1655                 bytes = ALIGN(ilsize, sizeof(struct sq_sge));
1656                 bytes += sizeof(struct sq_send_hdr);
1657         }
1658
1659         *qdf =  __xlate_qfd(qp->sq.q_full_delta, bytes);
1660         slot = bytes >> 4;
1661         *wqe_sz = slot;
1662         if (mode == BNXT_QPLIB_WQE_MODE_STATIC)
1663                 slot = 8;
1664         return slot;
1665 }
1666
1667 static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_q *sq,
1668                                      struct bnxt_qplib_swq *swq)
1669 {
1670         struct bnxt_qplib_hwq *hwq;
1671         u32 pg_num, pg_indx;
1672         void *buff;
1673         u32 tail;
1674
1675         hwq = &sq->hwq;
1676         if (!hwq->pad_pg)
1677                 return;
1678         tail = swq->slot_idx / sq->dbinfo.max_slot;
1679         pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride);
1680         pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride);
1681         buff = (void *)(hwq->pad_pg[pg_num] + pg_indx * hwq->pad_stride);
1682         swq->psn_ext = buff;
1683         swq->psn_search = buff;
1684 }
1685
1686 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1687 {
1688         struct bnxt_qplib_q *sq = &qp->sq;
1689
1690         bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
1691 }
1692
1693 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1694                          struct bnxt_qplib_swqe *wqe)
1695 {
1696         struct bnxt_qplib_nq_work *nq_work = NULL;
1697         int i, rc = 0, data_len = 0, pkt_num = 0;
1698         struct bnxt_qplib_q *sq = &qp->sq;
1699         struct bnxt_qplib_hwq *hwq;
1700         struct bnxt_qplib_swq *swq;
1701         bool sch_handler = false;
1702         u16 wqe_sz, qdf = 0;
1703         void *base_hdr;
1704         void *ext_hdr;
1705         __le32 temp32;
1706         u32 wqe_idx;
1707         u32 slots;
1708         u16 idx;
1709
1710         hwq = &sq->hwq;
1711         if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
1712             qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1713                 dev_err(&hwq->pdev->dev,
1714                         "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1715                         qp->id, qp->state);
1716                 rc = -EINVAL;
1717                 goto done;
1718         }
1719
1720         slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode);
1721         if (bnxt_qplib_queue_full(sq, slots + qdf)) {
1722                 dev_err(&hwq->pdev->dev,
1723                         "prod = %#x cons = %#x qdepth = %#x delta = %#x\n",
1724                         hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta);
1725                 rc = -ENOMEM;
1726                 goto done;
1727         }
1728
1729         swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
1730         bnxt_qplib_pull_psn_buff(sq, swq);
1731
1732         idx = 0;
1733         swq->slot_idx = hwq->prod;
1734         swq->slots = slots;
1735         swq->wr_id = wqe->wr_id;
1736         swq->type = wqe->type;
1737         swq->flags = wqe->flags;
1738         swq->start_psn = sq->psn & BTH_PSN_MASK;
1739         if (qp->sig_type)
1740                 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1741
1742         if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1743                 sch_handler = true;
1744                 dev_dbg(&hwq->pdev->dev,
1745                         "%s Error QP. Scheduling for poll_cq\n", __func__);
1746                 goto queue_err;
1747         }
1748
1749         base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
1750         ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
1751         memset(base_hdr, 0, sizeof(struct sq_sge));
1752         memset(ext_hdr, 0, sizeof(struct sq_sge));
1753
1754         if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
1755                 /* Copy the inline data */
1756                 data_len = bnxt_qplib_put_inline(qp, wqe, &idx);
1757         else
1758                 data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge,
1759                                                &idx);
1760         if (data_len < 0)
1761                 goto queue_err;
1762         /* Specifics */
1763         switch (wqe->type) {
1764         case BNXT_QPLIB_SWQE_TYPE_SEND:
1765                 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1766                         struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
1767                         struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
1768                         /* Assemble info for Raw Ethertype QPs */
1769
1770                         sqe->wqe_type = wqe->type;
1771                         sqe->flags = wqe->flags;
1772                         sqe->wqe_size = wqe_sz;
1773                         sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1774                         sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1775                         sqe->length = cpu_to_le32(data_len);
1776                         ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1777                                 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1778                                 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1779
1780                         break;
1781                 }
1782                 /* fall thru */
1783         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1784         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1785         {
1786                 struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
1787                 struct sq_send_hdr *sqe = base_hdr;
1788
1789                 sqe->wqe_type = wqe->type;
1790                 sqe->flags = wqe->flags;
1791                 sqe->wqe_size = wqe_sz;
1792                 sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
1793                 if (qp->type == CMDQ_CREATE_QP_TYPE_UD ||
1794                     qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
1795                         sqe->q_key = cpu_to_le32(wqe->send.q_key);
1796                         sqe->length = cpu_to_le32(data_len);
1797                         sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1798                         ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp &
1799                                                       SQ_SEND_DST_QP_MASK);
1800                         ext_sqe->avid = cpu_to_le32(wqe->send.avid &
1801                                                     SQ_SEND_AVID_MASK);
1802                 } else {
1803                         sqe->length = cpu_to_le32(data_len);
1804                         if (qp->mtu)
1805                                 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1806                         if (!pkt_num)
1807                                 pkt_num = 1;
1808                         sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1809                 }
1810                 break;
1811         }
1812         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1813         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1814         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1815         {
1816                 struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
1817                 struct sq_rdma_hdr *sqe = base_hdr;
1818
1819                 sqe->wqe_type = wqe->type;
1820                 sqe->flags = wqe->flags;
1821                 sqe->wqe_size = wqe_sz;
1822                 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1823                 sqe->length = cpu_to_le32((u32)data_len);
1824                 ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1825                 ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1826                 if (qp->mtu)
1827                         pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1828                 if (!pkt_num)
1829                         pkt_num = 1;
1830                 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1831                 break;
1832         }
1833         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1834         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1835         {
1836                 struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
1837                 struct sq_atomic_hdr *sqe = base_hdr;
1838
1839                 sqe->wqe_type = wqe->type;
1840                 sqe->flags = wqe->flags;
1841                 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1842                 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1843                 ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1844                 ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1845                 if (qp->mtu)
1846                         pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1847                 if (!pkt_num)
1848                         pkt_num = 1;
1849                 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1850                 break;
1851         }
1852         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1853         {
1854                 struct sq_localinvalidate *sqe = base_hdr;
1855
1856                 sqe->wqe_type = wqe->type;
1857                 sqe->flags = wqe->flags;
1858                 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1859
1860                 break;
1861         }
1862         case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1863         {
1864                 struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
1865                 struct sq_fr_pmr_hdr *sqe = base_hdr;
1866
1867                 sqe->wqe_type = wqe->type;
1868                 sqe->flags = wqe->flags;
1869                 sqe->access_cntl = wqe->frmr.access_cntl |
1870                                    SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1871                 sqe->zero_based_page_size_log =
1872                         (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1873                         SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1874                         (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1875                 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1876                 temp32 = cpu_to_le32(wqe->frmr.length);
1877                 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1878                 sqe->numlevels_pbl_page_size_log =
1879                         ((wqe->frmr.pbl_pg_sz_log <<
1880                                         SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1881                                         SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1882                         ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1883                                         SQ_FR_PMR_NUMLEVELS_MASK);
1884
1885                 for (i = 0; i < wqe->frmr.page_list_len; i++)
1886                         wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1887                                                 wqe->frmr.page_list[i] |
1888                                                 PTU_PTE_VALID);
1889                 ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1890                 ext_sqe->va = cpu_to_le64(wqe->frmr.va);
1891
1892                 break;
1893         }
1894         case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1895         {
1896                 struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
1897                 struct sq_bind_hdr *sqe = base_hdr;
1898
1899                 sqe->wqe_type = wqe->type;
1900                 sqe->flags = wqe->flags;
1901                 sqe->access_cntl = wqe->bind.access_cntl;
1902                 sqe->mw_type_zero_based = wqe->bind.mw_type |
1903                         (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1904                 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1905                 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1906                 ext_sqe->va = cpu_to_le64(wqe->bind.va);
1907                 ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
1908                 break;
1909         }
1910         default:
1911                 /* Bad wqe, return error */
1912                 rc = -EINVAL;
1913                 goto done;
1914         }
1915         swq->next_psn = sq->psn & BTH_PSN_MASK;
1916         bnxt_qplib_fill_psn_search(qp, wqe, swq);
1917 queue_err:
1918         bnxt_qplib_swq_mod_start(sq, wqe_idx);
1919         bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
1920         qp->wqe_cnt++;
1921 done:
1922         if (sch_handler) {
1923                 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1924                 if (nq_work) {
1925                         nq_work->cq = qp->scq;
1926                         nq_work->nq = qp->scq->nq;
1927                         INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1928                         queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1929                 } else {
1930                         dev_err(&hwq->pdev->dev,
1931                                 "FP: Failed to allocate SQ nq_work!\n");
1932                         rc = -ENOMEM;
1933                 }
1934         }
1935         return rc;
1936 }
1937
1938 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1939 {
1940         struct bnxt_qplib_q *rq = &qp->rq;
1941
1942         bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
1943 }
1944
1945 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1946                          struct bnxt_qplib_swqe *wqe)
1947 {
1948         struct bnxt_qplib_nq_work *nq_work = NULL;
1949         struct bnxt_qplib_q *rq = &qp->rq;
1950         struct rq_wqe_hdr *base_hdr;
1951         struct rq_ext_hdr *ext_hdr;
1952         struct bnxt_qplib_hwq *hwq;
1953         struct bnxt_qplib_swq *swq;
1954         bool sch_handler = false;
1955         u16 wqe_sz, idx;
1956         u32 wqe_idx;
1957         int rc = 0;
1958
1959         hwq = &rq->hwq;
1960         if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1961                 dev_err(&hwq->pdev->dev,
1962                         "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1963                         qp->id, qp->state);
1964                 rc = -EINVAL;
1965                 goto done;
1966         }
1967
1968         if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) {
1969                 dev_err(&hwq->pdev->dev,
1970                         "FP: QP (0x%x) RQ is full!\n", qp->id);
1971                 rc = -EINVAL;
1972                 goto done;
1973         }
1974
1975         swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
1976         swq->wr_id = wqe->wr_id;
1977         swq->slots = rq->dbinfo.max_slot;
1978
1979         if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1980                 sch_handler = true;
1981                 dev_dbg(&hwq->pdev->dev,
1982                         "%s: Error QP. Scheduling for poll_cq\n", __func__);
1983                 goto queue_err;
1984         }
1985
1986         idx = 0;
1987         base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
1988         ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
1989         memset(base_hdr, 0, sizeof(struct sq_sge));
1990         memset(ext_hdr, 0, sizeof(struct sq_sge));
1991         wqe_sz = (sizeof(struct rq_wqe_hdr) +
1992         wqe->num_sge * sizeof(struct sq_sge)) >> 4;
1993         bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx);
1994         if (!wqe->num_sge) {
1995                 struct sq_sge *sge;
1996
1997                 sge = bnxt_qplib_get_prod_qe(hwq, idx++);
1998                 sge->size = 0;
1999                 wqe_sz++;
2000         }
2001         base_hdr->wqe_type = wqe->type;
2002         base_hdr->flags = wqe->flags;
2003         base_hdr->wqe_size = wqe_sz;
2004         base_hdr->wr_id[0] = cpu_to_le32(wqe_idx);
2005 queue_err:
2006         bnxt_qplib_swq_mod_start(rq, wqe_idx);
2007         bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
2008 done:
2009         if (sch_handler) {
2010                 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
2011                 if (nq_work) {
2012                         nq_work->cq = qp->rcq;
2013                         nq_work->nq = qp->rcq->nq;
2014                         INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
2015                         queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
2016                 } else {
2017                         dev_err(&hwq->pdev->dev,
2018                                 "FP: Failed to allocate RQ nq_work!\n");
2019                         rc = -ENOMEM;
2020                 }
2021         }
2022
2023         return rc;
2024 }
2025
2026 /* CQ */
2027 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
2028 {
2029         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
2030         struct bnxt_qplib_hwq_attr hwq_attr = {};
2031         struct creq_create_cq_resp resp;
2032         struct bnxt_qplib_pbl *pbl;
2033         struct cmdq_create_cq req;
2034         u16 cmd_flags = 0;
2035         u32 pg_sz_lvl;
2036         int rc;
2037
2038         hwq_attr.res = res;
2039         hwq_attr.depth = cq->max_wqe;
2040         hwq_attr.stride = sizeof(struct cq_base);
2041         hwq_attr.type = HWQ_TYPE_QUEUE;
2042         hwq_attr.sginfo = &cq->sg_info;
2043         rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
2044         if (rc)
2045                 goto exit;
2046
2047         RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
2048
2049         if (!cq->dpi) {
2050                 dev_err(&rcfw->pdev->dev,
2051                         "FP: CREATE_CQ failed due to NULL DPI\n");
2052                 return -EINVAL;
2053         }
2054         req.dpi = cpu_to_le32(cq->dpi->dpi);
2055         req.cq_handle = cpu_to_le64(cq->cq_handle);
2056         req.cq_size = cpu_to_le32(cq->hwq.max_elements);
2057         pbl = &cq->hwq.pbl[PBL_LVL_0];
2058         pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) <<
2059                      CMDQ_CREATE_CQ_PG_SIZE_SFT);
2060         pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK);
2061         req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
2062         req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
2063         req.cq_fco_cnq_id = cpu_to_le32(
2064                         (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
2065                          CMDQ_CREATE_CQ_CNQ_ID_SFT);
2066
2067         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
2068                                           (void *)&resp, NULL, 0);
2069         if (rc)
2070                 goto fail;
2071
2072         cq->id = le32_to_cpu(resp.xid);
2073         cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
2074         init_waitqueue_head(&cq->waitq);
2075         INIT_LIST_HEAD(&cq->sqf_head);
2076         INIT_LIST_HEAD(&cq->rqf_head);
2077         spin_lock_init(&cq->compl_lock);
2078         spin_lock_init(&cq->flush_lock);
2079
2080         cq->dbinfo.hwq = &cq->hwq;
2081         cq->dbinfo.xid = cq->id;
2082         cq->dbinfo.db = cq->dpi->dbr;
2083         cq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
2084
2085         bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
2086
2087         return 0;
2088
2089 fail:
2090         bnxt_qplib_free_hwq(res, &cq->hwq);
2091 exit:
2092         return rc;
2093 }
2094
2095 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
2096 {
2097         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
2098         struct cmdq_destroy_cq req;
2099         struct creq_destroy_cq_resp resp;
2100         u16 total_cnq_events;
2101         u16 cmd_flags = 0;
2102         int rc;
2103
2104         RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
2105
2106         req.cq_cid = cpu_to_le32(cq->id);
2107         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
2108                                           (void *)&resp, NULL, 0);
2109         if (rc)
2110                 return rc;
2111         total_cnq_events = le16_to_cpu(resp.total_cnq_events);
2112         __wait_for_all_nqes(cq, total_cnq_events);
2113         bnxt_qplib_free_hwq(res, &cq->hwq);
2114         return 0;
2115 }
2116
2117 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
2118                       struct bnxt_qplib_cqe **pcqe, int *budget)
2119 {
2120         struct bnxt_qplib_cqe *cqe;
2121         u32 start, last;
2122         int rc = 0;
2123
2124         /* Now complete all outstanding SQEs with FLUSHED_ERR */
2125         start = sq->swq_start;
2126         cqe = *pcqe;
2127         while (*budget) {
2128                 last = sq->swq_last;
2129                 if (start == last)
2130                         break;
2131                 /* Skip the FENCE WQE completions */
2132                 if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
2133                         bnxt_qplib_cancel_phantom_processing(qp);
2134                         goto skip_compl;
2135                 }
2136                 memset(cqe, 0, sizeof(*cqe));
2137                 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
2138                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2139                 cqe->qp_handle = (u64)(unsigned long)qp;
2140                 cqe->wr_id = sq->swq[last].wr_id;
2141                 cqe->src_qp = qp->id;
2142                 cqe->type = sq->swq[last].type;
2143                 cqe++;
2144                 (*budget)--;
2145 skip_compl:
2146                 bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots);
2147                 sq->swq_last = sq->swq[last].next_idx;
2148         }
2149         *pcqe = cqe;
2150         if (!(*budget) && sq->swq_last != start)
2151                 /* Out of budget */
2152                 rc = -EAGAIN;
2153
2154         return rc;
2155 }
2156
2157 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
2158                       struct bnxt_qplib_cqe **pcqe, int *budget)
2159 {
2160         struct bnxt_qplib_cqe *cqe;
2161         u32 start, last;
2162         int opcode = 0;
2163         int rc = 0;
2164
2165         switch (qp->type) {
2166         case CMDQ_CREATE_QP1_TYPE_GSI:
2167                 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2168                 break;
2169         case CMDQ_CREATE_QP_TYPE_RC:
2170                 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2171                 break;
2172         case CMDQ_CREATE_QP_TYPE_UD:
2173         case CMDQ_CREATE_QP_TYPE_GSI:
2174                 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2175                 break;
2176         }
2177
2178         /* Flush the rest of the RQ */
2179         start = rq->swq_start;
2180         cqe = *pcqe;
2181         while (*budget) {
2182                 last = rq->swq_last;
2183                 if (last == start)
2184                         break;
2185                 memset(cqe, 0, sizeof(*cqe));
2186                 cqe->status =
2187                     CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2188                 cqe->opcode = opcode;
2189                 cqe->qp_handle = (unsigned long)qp;
2190                 cqe->wr_id = rq->swq[last].wr_id;
2191                 cqe++;
2192                 (*budget)--;
2193                 bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots);
2194                 rq->swq_last = rq->swq[last].next_idx;
2195         }
2196         *pcqe = cqe;
2197         if (!*budget && rq->swq_last != start)
2198                 /* Out of budget */
2199                 rc = -EAGAIN;
2200
2201         return rc;
2202 }
2203
2204 void bnxt_qplib_mark_qp_error(void *qp_handle)
2205 {
2206         struct bnxt_qplib_qp *qp = qp_handle;
2207
2208         if (!qp)
2209                 return;
2210
2211         /* Must block new posting of SQ and RQ */
2212         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2213         bnxt_qplib_cancel_phantom_processing(qp);
2214 }
2215
2216 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
2217  *       CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2218  */
2219 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
2220                      u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)
2221 {
2222         u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
2223         struct bnxt_qplib_q *sq = &qp->sq;
2224         struct cq_req *peek_req_hwcqe;
2225         struct bnxt_qplib_qp *peek_qp;
2226         struct bnxt_qplib_q *peek_sq;
2227         struct bnxt_qplib_swq *swq;
2228         struct cq_base *peek_hwcqe;
2229         int i, rc = 0;
2230
2231         /* Normal mode */
2232         /* Check for the psn_search marking before completing */
2233         swq = &sq->swq[swq_last];
2234         if (swq->psn_search &&
2235             le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2236                 /* Unmark */
2237                 swq->psn_search->flags_next_psn = cpu_to_le32
2238                         (le32_to_cpu(swq->psn_search->flags_next_psn)
2239                                      & ~0x80000000);
2240                 dev_dbg(&cq->hwq.pdev->dev,
2241                         "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2242                         cq_cons, qp->id, swq_last, cqe_sq_cons);
2243                 sq->condition = true;
2244                 sq->send_phantom = true;
2245
2246                 /* TODO: Only ARM if the previous SQE is ARMALL */
2247                 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
2248                 rc = -EAGAIN;
2249                 goto out;
2250         }
2251         if (sq->condition) {
2252                 /* Peek at the completions */
2253                 peek_raw_cq_cons = cq->hwq.cons;
2254                 peek_sw_cq_cons = cq_cons;
2255                 i = cq->hwq.max_elements;
2256                 while (i--) {
2257                         peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
2258                         peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
2259                                                        peek_sw_cq_cons, NULL);
2260                         /* If the next hwcqe is VALID */
2261                         if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
2262                                           cq->hwq.max_elements)) {
2263                         /*
2264                          * The valid test of the entry must be done first before
2265                          * reading any further.
2266                          */
2267                                 dma_rmb();
2268                                 /* If the next hwcqe is a REQ */
2269                                 if ((peek_hwcqe->cqe_type_toggle &
2270                                     CQ_BASE_CQE_TYPE_MASK) ==
2271                                     CQ_BASE_CQE_TYPE_REQ) {
2272                                         peek_req_hwcqe = (struct cq_req *)
2273                                                          peek_hwcqe;
2274                                         peek_qp = (struct bnxt_qplib_qp *)
2275                                                 ((unsigned long)
2276                                                  le64_to_cpu
2277                                                  (peek_req_hwcqe->qp_handle));
2278                                         peek_sq = &peek_qp->sq;
2279                                         peek_sq_cons_idx =
2280                                                 ((le16_to_cpu(
2281                                                   peek_req_hwcqe->sq_cons_idx)
2282                                                   - 1) % sq->max_wqe);
2283                                         /* If the hwcqe's sq's wr_id matches */
2284                                         if (peek_sq == sq &&
2285                                             sq->swq[peek_sq_cons_idx].wr_id ==
2286                                             BNXT_QPLIB_FENCE_WRID) {
2287                                                 /*
2288                                                  *  Unbreak only if the phantom
2289                                                  *  comes back
2290                                                  */
2291                                                 dev_dbg(&cq->hwq.pdev->dev,
2292                                                         "FP: Got Phantom CQE\n");
2293                                                 sq->condition = false;
2294                                                 sq->single = true;
2295                                                 rc = 0;
2296                                                 goto out;
2297                                         }
2298                                 }
2299                                 /* Valid but not the phantom, so keep looping */
2300                         } else {
2301                                 /* Not valid yet, just exit and wait */
2302                                 rc = -EINVAL;
2303                                 goto out;
2304                         }
2305                         peek_sw_cq_cons++;
2306                         peek_raw_cq_cons++;
2307                 }
2308                 dev_err(&cq->hwq.pdev->dev,
2309                         "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
2310                         cq_cons, qp->id, swq_last, cqe_sq_cons);
2311                 rc = -EINVAL;
2312         }
2313 out:
2314         return rc;
2315 }
2316
2317 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
2318                                      struct cq_req *hwcqe,
2319                                      struct bnxt_qplib_cqe **pcqe, int *budget,
2320                                      u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
2321 {
2322         struct bnxt_qplib_swq *swq;
2323         struct bnxt_qplib_cqe *cqe;
2324         struct bnxt_qplib_qp *qp;
2325         struct bnxt_qplib_q *sq;
2326         u32 cqe_sq_cons;
2327         int rc = 0;
2328
2329         qp = (struct bnxt_qplib_qp *)((unsigned long)
2330                                       le64_to_cpu(hwcqe->qp_handle));
2331         if (!qp) {
2332                 dev_err(&cq->hwq.pdev->dev,
2333                         "FP: Process Req qp is NULL\n");
2334                 return -EINVAL;
2335         }
2336         sq = &qp->sq;
2337
2338         cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
2339         if (qp->sq.flushed) {
2340                 dev_dbg(&cq->hwq.pdev->dev,
2341                         "%s: QP in Flush QP = %p\n", __func__, qp);
2342                 goto done;
2343         }
2344         /* Require to walk the sq's swq to fabricate CQEs for all previously
2345          * signaled SWQEs due to CQE aggregation from the current sq cons
2346          * to the cqe_sq_cons
2347          */
2348         cqe = *pcqe;
2349         while (*budget) {
2350                 if (sq->swq_last == cqe_sq_cons)
2351                         /* Done */
2352                         break;
2353
2354                 swq = &sq->swq[sq->swq_last];
2355                 memset(cqe, 0, sizeof(*cqe));
2356                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2357                 cqe->qp_handle = (u64)(unsigned long)qp;
2358                 cqe->src_qp = qp->id;
2359                 cqe->wr_id = swq->wr_id;
2360                 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
2361                         goto skip;
2362                 cqe->type = swq->type;
2363
2364                 /* For the last CQE, check for status.  For errors, regardless
2365                  * of the request being signaled or not, it must complete with
2366                  * the hwcqe error status
2367                  */
2368                 if (swq->next_idx == cqe_sq_cons &&
2369                     hwcqe->status != CQ_REQ_STATUS_OK) {
2370                         cqe->status = hwcqe->status;
2371                         dev_err(&cq->hwq.pdev->dev,
2372                                 "FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",
2373                                 sq->swq_last, cqe->wr_id, cqe->status);
2374                         cqe++;
2375                         (*budget)--;
2376                         bnxt_qplib_mark_qp_error(qp);
2377                         /* Add qp to flush list of the CQ */
2378                         bnxt_qplib_add_flush_qp(qp);
2379                 } else {
2380                         /* Before we complete, do WA 9060 */
2381                         if (do_wa9060(qp, cq, cq_cons, sq->swq_last,
2382                                       cqe_sq_cons)) {
2383                                 *lib_qp = qp;
2384                                 goto out;
2385                         }
2386                         if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2387                                 cqe->status = CQ_REQ_STATUS_OK;
2388                                 cqe++;
2389                                 (*budget)--;
2390                         }
2391                 }
2392 skip:
2393                 bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots);
2394                 sq->swq_last = swq->next_idx;
2395                 if (sq->single)
2396                         break;
2397         }
2398 out:
2399         *pcqe = cqe;
2400         if (sq->swq_last != cqe_sq_cons) {
2401                 /* Out of budget */
2402                 rc = -EAGAIN;
2403                 goto done;
2404         }
2405         /*
2406          * Back to normal completion mode only after it has completed all of
2407          * the WC for this CQE
2408          */
2409         sq->single = false;
2410 done:
2411         return rc;
2412 }
2413
2414 static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
2415 {
2416         spin_lock(&srq->hwq.lock);
2417         srq->swq[srq->last_idx].next_idx = (int)tag;
2418         srq->last_idx = (int)tag;
2419         srq->swq[srq->last_idx].next_idx = -1;
2420         srq->hwq.cons++; /* Support for SRQE counter */
2421         spin_unlock(&srq->hwq.lock);
2422 }
2423
2424 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
2425                                         struct cq_res_rc *hwcqe,
2426                                         struct bnxt_qplib_cqe **pcqe,
2427                                         int *budget)
2428 {
2429         struct bnxt_qplib_srq *srq;
2430         struct bnxt_qplib_cqe *cqe;
2431         struct bnxt_qplib_qp *qp;
2432         struct bnxt_qplib_q *rq;
2433         u32 wr_id_idx;
2434         int rc = 0;
2435
2436         qp = (struct bnxt_qplib_qp *)((unsigned long)
2437                                       le64_to_cpu(hwcqe->qp_handle));
2438         if (!qp) {
2439                 dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n");
2440                 return -EINVAL;
2441         }
2442         if (qp->rq.flushed) {
2443                 dev_dbg(&cq->hwq.pdev->dev,
2444                         "%s: QP in Flush QP = %p\n", __func__, qp);
2445                 goto done;
2446         }
2447
2448         cqe = *pcqe;
2449         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2450         cqe->length = le32_to_cpu(hwcqe->length);
2451         cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
2452         cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
2453         cqe->flags = le16_to_cpu(hwcqe->flags);
2454         cqe->status = hwcqe->status;
2455         cqe->qp_handle = (u64)(unsigned long)qp;
2456
2457         wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
2458                                 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
2459         if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
2460                 srq = qp->srq;
2461                 if (!srq)
2462                         return -EINVAL;
2463                 if (wr_id_idx >= srq->hwq.max_elements) {
2464                         dev_err(&cq->hwq.pdev->dev,
2465                                 "FP: CQ Process RC wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2466                                 wr_id_idx, srq->hwq.max_elements);
2467                         return -EINVAL;
2468                 }
2469                 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2470                 bnxt_qplib_release_srqe(srq, wr_id_idx);
2471                 cqe++;
2472                 (*budget)--;
2473                 *pcqe = cqe;
2474         } else {
2475                 struct bnxt_qplib_swq *swq;
2476
2477                 rq = &qp->rq;
2478                 if (wr_id_idx > (rq->max_wqe - 1)) {
2479                         dev_err(&cq->hwq.pdev->dev,
2480                                 "FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",
2481                                 wr_id_idx, rq->max_wqe);
2482                         return -EINVAL;
2483                 }
2484                 if (wr_id_idx != rq->swq_last)
2485                         return -EINVAL;
2486                 swq = &rq->swq[rq->swq_last];
2487                 cqe->wr_id = swq->wr_id;
2488                 cqe++;
2489                 (*budget)--;
2490                 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
2491                 rq->swq_last = swq->next_idx;
2492                 *pcqe = cqe;
2493
2494                 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2495                         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2496                         /* Add qp to flush list of the CQ */
2497                         bnxt_qplib_add_flush_qp(qp);
2498                 }
2499         }
2500
2501 done:
2502         return rc;
2503 }
2504
2505 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
2506                                         struct cq_res_ud *hwcqe,
2507                                         struct bnxt_qplib_cqe **pcqe,
2508                                         int *budget)
2509 {
2510         struct bnxt_qplib_srq *srq;
2511         struct bnxt_qplib_cqe *cqe;
2512         struct bnxt_qplib_qp *qp;
2513         struct bnxt_qplib_q *rq;
2514         u32 wr_id_idx;
2515         int rc = 0;
2516
2517         qp = (struct bnxt_qplib_qp *)((unsigned long)
2518                                       le64_to_cpu(hwcqe->qp_handle));
2519         if (!qp) {
2520                 dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n");
2521                 return -EINVAL;
2522         }
2523         if (qp->rq.flushed) {
2524                 dev_dbg(&cq->hwq.pdev->dev,
2525                         "%s: QP in Flush QP = %p\n", __func__, qp);
2526                 goto done;
2527         }
2528         cqe = *pcqe;
2529         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2530         cqe->length = le16_to_cpu(hwcqe->length) & CQ_RES_UD_LENGTH_MASK;
2531         cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata);
2532         cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
2533         cqe->flags = le16_to_cpu(hwcqe->flags);
2534         cqe->status = hwcqe->status;
2535         cqe->qp_handle = (u64)(unsigned long)qp;
2536         /*FIXME: Endianness fix needed for smace */
2537         memcpy(cqe->smac, hwcqe->src_mac, ETH_ALEN);
2538         wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
2539                                 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
2540         cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
2541                                   ((le32_to_cpu(
2542                                   hwcqe->src_qp_high_srq_or_rq_wr_id) &
2543                                  CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
2544
2545         if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
2546                 srq = qp->srq;
2547                 if (!srq)
2548                         return -EINVAL;
2549
2550                 if (wr_id_idx >= srq->hwq.max_elements) {
2551                         dev_err(&cq->hwq.pdev->dev,
2552                                 "FP: CQ Process UD wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2553                                 wr_id_idx, srq->hwq.max_elements);
2554                         return -EINVAL;
2555                 }
2556                 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2557                 bnxt_qplib_release_srqe(srq, wr_id_idx);
2558                 cqe++;
2559                 (*budget)--;
2560                 *pcqe = cqe;
2561         } else {
2562                 struct bnxt_qplib_swq *swq;
2563
2564                 rq = &qp->rq;
2565                 if (wr_id_idx > (rq->max_wqe - 1)) {
2566                         dev_err(&cq->hwq.pdev->dev,
2567                                 "FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",
2568                                 wr_id_idx, rq->max_wqe);
2569                         return -EINVAL;
2570                 }
2571
2572                 if (rq->swq_last != wr_id_idx)
2573                         return -EINVAL;
2574                 swq = &rq->swq[rq->swq_last];
2575                 cqe->wr_id = swq->wr_id;
2576                 cqe++;
2577                 (*budget)--;
2578                 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
2579                 rq->swq_last = swq->next_idx;
2580                 *pcqe = cqe;
2581
2582                 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2583                         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2584                         /* Add qp to flush list of the CQ */
2585                         bnxt_qplib_add_flush_qp(qp);
2586                 }
2587         }
2588 done:
2589         return rc;
2590 }
2591
2592 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
2593 {
2594         struct cq_base *hw_cqe;
2595         u32 sw_cons, raw_cons;
2596         bool rc = true;
2597
2598         raw_cons = cq->hwq.cons;
2599         sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2600         hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
2601          /* Check for Valid bit. If the CQE is valid, return false */
2602         rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
2603         return rc;
2604 }
2605
2606 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
2607                                                 struct cq_res_raweth_qp1 *hwcqe,
2608                                                 struct bnxt_qplib_cqe **pcqe,
2609                                                 int *budget)
2610 {
2611         struct bnxt_qplib_qp *qp;
2612         struct bnxt_qplib_q *rq;
2613         struct bnxt_qplib_srq *srq;
2614         struct bnxt_qplib_cqe *cqe;
2615         u32 wr_id_idx;
2616         int rc = 0;
2617
2618         qp = (struct bnxt_qplib_qp *)((unsigned long)
2619                                       le64_to_cpu(hwcqe->qp_handle));
2620         if (!qp) {
2621                 dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n");
2622                 return -EINVAL;
2623         }
2624         if (qp->rq.flushed) {
2625                 dev_dbg(&cq->hwq.pdev->dev,
2626                         "%s: QP in Flush QP = %p\n", __func__, qp);
2627                 goto done;
2628         }
2629         cqe = *pcqe;
2630         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2631         cqe->flags = le16_to_cpu(hwcqe->flags);
2632         cqe->qp_handle = (u64)(unsigned long)qp;
2633
2634         wr_id_idx =
2635                 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
2636                                 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
2637         cqe->src_qp = qp->id;
2638         if (qp->id == 1 && !cqe->length) {
2639                 /* Add workaround for the length misdetection */
2640                 cqe->length = 296;
2641         } else {
2642                 cqe->length = le16_to_cpu(hwcqe->length);
2643         }
2644         cqe->pkey_index = qp->pkey_index;
2645         memcpy(cqe->smac, qp->smac, 6);
2646
2647         cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
2648         cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
2649         cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
2650
2651         if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
2652                 srq = qp->srq;
2653                 if (!srq) {
2654                         dev_err(&cq->hwq.pdev->dev,
2655                                 "FP: SRQ used but not defined??\n");
2656                         return -EINVAL;
2657                 }
2658                 if (wr_id_idx >= srq->hwq.max_elements) {
2659                         dev_err(&cq->hwq.pdev->dev,
2660                                 "FP: CQ Process Raw/QP1 wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2661                                 wr_id_idx, srq->hwq.max_elements);
2662                         return -EINVAL;
2663                 }
2664                 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2665                 bnxt_qplib_release_srqe(srq, wr_id_idx);
2666                 cqe++;
2667                 (*budget)--;
2668                 *pcqe = cqe;
2669         } else {
2670                 struct bnxt_qplib_swq *swq;
2671
2672                 rq = &qp->rq;
2673                 if (wr_id_idx > (rq->max_wqe - 1)) {
2674                         dev_err(&cq->hwq.pdev->dev,
2675                                 "FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",
2676                                 wr_id_idx, rq->max_wqe);
2677                         return -EINVAL;
2678                 }
2679                 if (rq->swq_last != wr_id_idx)
2680                         return -EINVAL;
2681                 swq = &rq->swq[rq->swq_last];
2682                 cqe->wr_id = swq->wr_id;
2683                 cqe++;
2684                 (*budget)--;
2685                 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
2686                 rq->swq_last = swq->next_idx;
2687                 *pcqe = cqe;
2688
2689                 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2690                         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2691                         /* Add qp to flush list of the CQ */
2692                         bnxt_qplib_add_flush_qp(qp);
2693                 }
2694         }
2695
2696 done:
2697         return rc;
2698 }
2699
2700 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
2701                                           struct cq_terminal *hwcqe,
2702                                           struct bnxt_qplib_cqe **pcqe,
2703                                           int *budget)
2704 {
2705         struct bnxt_qplib_qp *qp;
2706         struct bnxt_qplib_q *sq, *rq;
2707         struct bnxt_qplib_cqe *cqe;
2708         u32 swq_last = 0, cqe_cons;
2709         int rc = 0;
2710
2711         /* Check the Status */
2712         if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
2713                 dev_warn(&cq->hwq.pdev->dev,
2714                          "FP: CQ Process Terminal Error status = 0x%x\n",
2715                          hwcqe->status);
2716
2717         qp = (struct bnxt_qplib_qp *)((unsigned long)
2718                                       le64_to_cpu(hwcqe->qp_handle));
2719         if (!qp) {
2720                 dev_err(&cq->hwq.pdev->dev,
2721                         "FP: CQ Process terminal qp is NULL\n");
2722                 return -EINVAL;
2723         }
2724
2725         /* Must block new posting of SQ and RQ */
2726         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2727
2728         sq = &qp->sq;
2729         rq = &qp->rq;
2730
2731         cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2732         if (cqe_cons == 0xFFFF)
2733                 goto do_rq;
2734         cqe_cons %= sq->max_wqe;
2735
2736         if (qp->sq.flushed) {
2737                 dev_dbg(&cq->hwq.pdev->dev,
2738                         "%s: QP in Flush QP = %p\n", __func__, qp);
2739                 goto sq_done;
2740         }
2741
2742         /* Terminal CQE can also include aggregated successful CQEs prior.
2743          * So we must complete all CQEs from the current sq's cons to the
2744          * cq_cons with status OK
2745          */
2746         cqe = *pcqe;
2747         while (*budget) {
2748                 swq_last = sq->swq_last;
2749                 if (swq_last == cqe_cons)
2750                         break;
2751                 if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2752                         memset(cqe, 0, sizeof(*cqe));
2753                         cqe->status = CQ_REQ_STATUS_OK;
2754                         cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2755                         cqe->qp_handle = (u64)(unsigned long)qp;
2756                         cqe->src_qp = qp->id;
2757                         cqe->wr_id = sq->swq[swq_last].wr_id;
2758                         cqe->type = sq->swq[swq_last].type;
2759                         cqe++;
2760                         (*budget)--;
2761                 }
2762                 bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots);
2763                 sq->swq_last = sq->swq[swq_last].next_idx;
2764         }
2765         *pcqe = cqe;
2766         if (!(*budget) && swq_last != cqe_cons) {
2767                 /* Out of budget */
2768                 rc = -EAGAIN;
2769                 goto sq_done;
2770         }
2771 sq_done:
2772         if (rc)
2773                 return rc;
2774 do_rq:
2775         cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2776         if (cqe_cons == 0xFFFF) {
2777                 goto done;
2778         } else if (cqe_cons > rq->max_wqe - 1) {
2779                 dev_err(&cq->hwq.pdev->dev,
2780                         "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",
2781                         cqe_cons, rq->max_wqe);
2782                 goto done;
2783         }
2784
2785         if (qp->rq.flushed) {
2786                 dev_dbg(&cq->hwq.pdev->dev,
2787                         "%s: QP in Flush QP = %p\n", __func__, qp);
2788                 rc = 0;
2789                 goto done;
2790         }
2791
2792         /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2793          * from the current rq->cons to the rq->prod regardless what the
2794          * rq->cons the terminal CQE indicates
2795          */
2796
2797         /* Add qp to flush list of the CQ */
2798         bnxt_qplib_add_flush_qp(qp);
2799 done:
2800         return rc;
2801 }
2802
2803 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2804                                         struct cq_cutoff *hwcqe)
2805 {
2806         /* Check the Status */
2807         if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2808                 dev_err(&cq->hwq.pdev->dev,
2809                         "FP: CQ Process Cutoff Error status = 0x%x\n",
2810                         hwcqe->status);
2811                 return -EINVAL;
2812         }
2813         clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2814         wake_up_interruptible(&cq->waitq);
2815
2816         return 0;
2817 }
2818
2819 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
2820                                   struct bnxt_qplib_cqe *cqe,
2821                                   int num_cqes)
2822 {
2823         struct bnxt_qplib_qp *qp = NULL;
2824         u32 budget = num_cqes;
2825         unsigned long flags;
2826
2827         spin_lock_irqsave(&cq->flush_lock, flags);
2828         list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
2829                 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp);
2830                 __flush_sq(&qp->sq, qp, &cqe, &budget);
2831         }
2832
2833         list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
2834                 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp);
2835                 __flush_rq(&qp->rq, qp, &cqe, &budget);
2836         }
2837         spin_unlock_irqrestore(&cq->flush_lock, flags);
2838
2839         return num_cqes - budget;
2840 }
2841
2842 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2843                        int num_cqes, struct bnxt_qplib_qp **lib_qp)
2844 {
2845         struct cq_base *hw_cqe;
2846         u32 sw_cons, raw_cons;
2847         int budget, rc = 0;
2848
2849         raw_cons = cq->hwq.cons;
2850         budget = num_cqes;
2851
2852         while (budget) {
2853                 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2854                 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
2855
2856                 /* Check for Valid bit */
2857                 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2858                         break;
2859
2860                 /*
2861                  * The valid test of the entry must be done first before
2862                  * reading any further.
2863                  */
2864                 dma_rmb();
2865                 /* From the device's respective CQE format to qplib_wc*/
2866                 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2867                 case CQ_BASE_CQE_TYPE_REQ:
2868                         rc = bnxt_qplib_cq_process_req(cq,
2869                                                        (struct cq_req *)hw_cqe,
2870                                                        &cqe, &budget,
2871                                                        sw_cons, lib_qp);
2872                         break;
2873                 case CQ_BASE_CQE_TYPE_RES_RC:
2874                         rc = bnxt_qplib_cq_process_res_rc(cq,
2875                                                           (struct cq_res_rc *)
2876                                                           hw_cqe, &cqe,
2877                                                           &budget);
2878                         break;
2879                 case CQ_BASE_CQE_TYPE_RES_UD:
2880                         rc = bnxt_qplib_cq_process_res_ud
2881                                         (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2882                                          &budget);
2883                         break;
2884                 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2885                         rc = bnxt_qplib_cq_process_res_raweth_qp1
2886                                         (cq, (struct cq_res_raweth_qp1 *)
2887                                          hw_cqe, &cqe, &budget);
2888                         break;
2889                 case CQ_BASE_CQE_TYPE_TERMINAL:
2890                         rc = bnxt_qplib_cq_process_terminal
2891                                         (cq, (struct cq_terminal *)hw_cqe,
2892                                          &cqe, &budget);
2893                         break;
2894                 case CQ_BASE_CQE_TYPE_CUT_OFF:
2895                         bnxt_qplib_cq_process_cutoff
2896                                         (cq, (struct cq_cutoff *)hw_cqe);
2897                         /* Done processing this CQ */
2898                         goto exit;
2899                 default:
2900                         dev_err(&cq->hwq.pdev->dev,
2901                                 "process_cq unknown type 0x%lx\n",
2902                                 hw_cqe->cqe_type_toggle &
2903                                 CQ_BASE_CQE_TYPE_MASK);
2904                         rc = -EINVAL;
2905                         break;
2906                 }
2907                 if (rc < 0) {
2908                         if (rc == -EAGAIN)
2909                                 break;
2910                         /* Error while processing the CQE, just skip to the
2911                          * next one
2912                          */
2913                         dev_err(&cq->hwq.pdev->dev,
2914                                 "process_cqe error rc = 0x%x\n", rc);
2915                 }
2916                 raw_cons++;
2917         }
2918         if (cq->hwq.cons != raw_cons) {
2919                 cq->hwq.cons = raw_cons;
2920                 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
2921         }
2922 exit:
2923         return num_cqes - budget;
2924 }
2925
2926 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2927 {
2928         if (arm_type)
2929                 bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
2930         /* Using cq->arm_state variable to track whether to issue cq handler */
2931         atomic_set(&cq->arm_state, 1);
2932 }
2933
2934 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
2935 {
2936         flush_workqueue(qp->scq->nq->cqn_wq);
2937         if (qp->scq != qp->rcq)
2938                 flush_workqueue(qp->rcq->nq->cqn_wq);
2939 }