1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
37 #include <linux/iommu.h>
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45 #include <rdma/uverbs_ioctl.h>
47 #include <linux/qed/common_hsi.h>
48 #include "qedr_hsi_rdma.h"
49 #include <linux/qed/qed_if.h>
52 #include <rdma/qedr-abi.h>
53 #include "qedr_roce_cm.h"
55 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
56 #define RDMA_MAX_SGE_PER_SRQ (4)
57 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
64 size_t min_len = min_t(size_t, len, udata->outlen);
66 return ib_copy_to_udata(udata, src, min_len);
69 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
71 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
74 *pkey = QEDR_ROCE_PKEY_DEFAULT;
78 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
79 int index, union ib_gid *sgid)
81 struct qedr_dev *dev = get_qedr_dev(ibdev);
83 memset(sgid->raw, 0, sizeof(sgid->raw));
84 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
86 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
87 sgid->global.interface_id, sgid->global.subnet_prefix);
92 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
94 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
95 struct qedr_device_attr *qattr = &dev->attr;
96 struct qedr_srq *srq = get_qedr_srq(ibsrq);
98 srq_attr->srq_limit = srq->srq_limit;
99 srq_attr->max_wr = qattr->max_srq_wr;
100 srq_attr->max_sge = qattr->max_sge;
105 int qedr_query_device(struct ib_device *ibdev,
106 struct ib_device_attr *attr, struct ib_udata *udata)
108 struct qedr_dev *dev = get_qedr_dev(ibdev);
109 struct qedr_device_attr *qattr = &dev->attr;
111 if (!dev->rdma_ctx) {
113 "qedr_query_device called with invalid params rdma_ctx=%p\n",
118 memset(attr, 0, sizeof(*attr));
120 attr->fw_ver = qattr->fw_ver;
121 attr->sys_image_guid = qattr->sys_image_guid;
122 attr->max_mr_size = qattr->max_mr_size;
123 attr->page_size_cap = qattr->page_size_caps;
124 attr->vendor_id = qattr->vendor_id;
125 attr->vendor_part_id = qattr->vendor_part_id;
126 attr->hw_ver = qattr->hw_ver;
127 attr->max_qp = qattr->max_qp;
128 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
129 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
130 IB_DEVICE_RC_RNR_NAK_GEN |
131 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
133 attr->max_send_sge = qattr->max_sge;
134 attr->max_recv_sge = qattr->max_sge;
135 attr->max_sge_rd = qattr->max_sge;
136 attr->max_cq = qattr->max_cq;
137 attr->max_cqe = qattr->max_cqe;
138 attr->max_mr = qattr->max_mr;
139 attr->max_mw = qattr->max_mw;
140 attr->max_pd = qattr->max_pd;
141 attr->atomic_cap = dev->atomic_cap;
142 attr->max_fmr = qattr->max_fmr;
143 attr->max_map_per_fmr = 16;
144 attr->max_qp_init_rd_atom =
145 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
146 attr->max_qp_rd_atom =
147 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
148 attr->max_qp_init_rd_atom);
150 attr->max_srq = qattr->max_srq;
151 attr->max_srq_sge = qattr->max_srq_sge;
152 attr->max_srq_wr = qattr->max_srq_wr;
154 attr->local_ca_ack_delay = qattr->dev_ack_delay;
155 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
156 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
157 attr->max_ah = qattr->max_ah;
162 #define QEDR_SPEED_SDR (1)
163 #define QEDR_SPEED_DDR (2)
164 #define QEDR_SPEED_QDR (4)
165 #define QEDR_SPEED_FDR10 (8)
166 #define QEDR_SPEED_FDR (16)
167 #define QEDR_SPEED_EDR (32)
169 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
174 *ib_speed = QEDR_SPEED_SDR;
175 *ib_width = IB_WIDTH_1X;
178 *ib_speed = QEDR_SPEED_QDR;
179 *ib_width = IB_WIDTH_1X;
183 *ib_speed = QEDR_SPEED_DDR;
184 *ib_width = IB_WIDTH_4X;
188 *ib_speed = QEDR_SPEED_EDR;
189 *ib_width = IB_WIDTH_1X;
193 *ib_speed = QEDR_SPEED_QDR;
194 *ib_width = IB_WIDTH_4X;
198 *ib_speed = QEDR_SPEED_QDR;
199 *ib_width = IB_WIDTH_4X;
203 *ib_speed = QEDR_SPEED_EDR;
204 *ib_width = IB_WIDTH_4X;
209 *ib_speed = QEDR_SPEED_SDR;
210 *ib_width = IB_WIDTH_1X;
214 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
216 struct qedr_dev *dev;
217 struct qed_rdma_port *rdma_port;
219 dev = get_qedr_dev(ibdev);
221 if (!dev->rdma_ctx) {
222 DP_ERR(dev, "rdma_ctx is NULL\n");
226 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
228 /* *attr being zeroed by the caller, avoid zeroing it here */
229 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
230 attr->state = IB_PORT_ACTIVE;
231 attr->phys_state = 5;
233 attr->state = IB_PORT_DOWN;
234 attr->phys_state = 3;
236 attr->max_mtu = IB_MTU_4096;
237 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
242 attr->ip_gids = true;
243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
244 attr->gid_tbl_len = 1;
245 attr->pkey_tbl_len = 1;
247 attr->gid_tbl_len = QEDR_MAX_SGID;
248 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
250 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
251 attr->qkey_viol_cntr = 0;
252 get_link_speed_and_width(rdma_port->link_speed,
253 &attr->active_speed, &attr->active_width);
254 attr->max_msg_sz = rdma_port->max_msg_size;
255 attr->max_vl_num = 4;
260 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
261 struct ib_port_modify *props)
266 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
271 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
275 mm->key.phy_addr = phy_addr;
276 /* This function might be called with a length which is not a multiple
277 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
278 * forces this granularity by increasing the requested size if needed.
279 * When qedr_mmap is called, it will search the list with the updated
280 * length as a key. To prevent search failures, the length is rounded up
281 * in advance to PAGE_SIZE.
283 mm->key.len = roundup(len, PAGE_SIZE);
284 INIT_LIST_HEAD(&mm->entry);
286 mutex_lock(&uctx->mm_list_lock);
287 list_add(&mm->entry, &uctx->mm_head);
288 mutex_unlock(&uctx->mm_list_lock);
290 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
291 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
292 (unsigned long long)mm->key.phy_addr,
293 (unsigned long)mm->key.len, uctx);
298 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
304 mutex_lock(&uctx->mm_list_lock);
305 list_for_each_entry(mm, &uctx->mm_head, entry) {
306 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
312 mutex_unlock(&uctx->mm_list_lock);
313 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
314 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
315 mm->key.phy_addr, mm->key.len, uctx, found);
320 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
322 struct ib_device *ibdev = uctx->device;
324 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
325 struct qedr_alloc_ucontext_resp uresp = {};
326 struct qedr_dev *dev = get_qedr_dev(ibdev);
327 struct qed_rdma_add_user_out_params oparams;
332 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
335 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
340 ctx->dpi = oparams.dpi;
341 ctx->dpi_addr = oparams.dpi_addr;
342 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
343 ctx->dpi_size = oparams.dpi_size;
344 INIT_LIST_HEAD(&ctx->mm_head);
345 mutex_init(&ctx->mm_list_lock);
347 uresp.dpm_enabled = dev->user_dpm_enabled;
348 uresp.wids_enabled = 1;
349 uresp.wid_count = oparams.wid_count;
350 uresp.db_pa = ctx->dpi_phys_addr;
351 uresp.db_size = ctx->dpi_size;
352 uresp.max_send_wr = dev->attr.max_sqe;
353 uresp.max_recv_wr = dev->attr.max_rqe;
354 uresp.max_srq_wr = dev->attr.max_srq_wr;
355 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
356 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
357 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
358 uresp.max_cqes = QEDR_MAX_CQES;
360 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
366 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
370 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
375 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
377 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
378 struct qedr_mm *mm, *tmp;
380 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
382 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
384 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
385 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
386 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
387 mm->key.phy_addr, mm->key.len, uctx);
388 list_del(&mm->entry);
393 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
395 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
396 struct qedr_dev *dev = get_qedr_dev(context->device);
397 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
398 unsigned long len = (vma->vm_end - vma->vm_start);
399 unsigned long dpi_start;
401 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
403 DP_DEBUG(dev, QEDR_MSG_INIT,
404 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
405 (void *)vma->vm_start, (void *)vma->vm_end,
406 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
408 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
410 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
411 (void *)vma->vm_start, (void *)vma->vm_end);
415 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
416 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
421 if (phys_addr < dpi_start ||
422 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
424 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
425 (void *)phys_addr, (void *)dpi_start,
430 if (vma->vm_flags & VM_READ) {
431 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
435 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
436 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
440 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
442 struct ib_device *ibdev = ibpd->device;
443 struct qedr_dev *dev = get_qedr_dev(ibdev);
444 struct qedr_pd *pd = get_qedr_pd(ibpd);
448 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
449 udata ? "User Lib" : "Kernel");
451 if (!dev->rdma_ctx) {
452 DP_ERR(dev, "invalid RDMA context\n");
456 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
463 struct qedr_alloc_pd_uresp uresp = {
466 struct qedr_ucontext *context = rdma_udata_to_drv_context(
467 udata, struct qedr_ucontext, ibucontext);
469 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
471 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
472 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
483 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
485 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
486 struct qedr_pd *pd = get_qedr_pd(ibpd);
488 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
489 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
492 static void qedr_free_pbl(struct qedr_dev *dev,
493 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
495 struct pci_dev *pdev = dev->pdev;
498 for (i = 0; i < pbl_info->num_pbls; i++) {
501 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
502 pbl[i].va, pbl[i].pa);
508 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
509 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
511 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
512 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
513 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
515 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
516 struct qedr_pbl_info *pbl_info,
519 struct pci_dev *pdev = dev->pdev;
520 struct qedr_pbl *pbl_table;
521 dma_addr_t *pbl_main_tbl;
526 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
528 return ERR_PTR(-ENOMEM);
530 for (i = 0; i < pbl_info->num_pbls; i++) {
531 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
536 pbl_table[i].va = va;
537 pbl_table[i].pa = pa;
540 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
541 * the first one with physical pointers to all of the rest
543 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
544 for (i = 0; i < pbl_info->num_pbls - 1; i++)
545 pbl_main_tbl[i] = pbl_table[i + 1].pa;
550 for (i--; i >= 0; i--)
551 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
552 pbl_table[i].va, pbl_table[i].pa);
554 qedr_free_pbl(dev, pbl_info, pbl_table);
556 return ERR_PTR(-ENOMEM);
559 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
560 struct qedr_pbl_info *pbl_info,
561 u32 num_pbes, int two_layer_capable)
567 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
568 if (num_pbes > MAX_PBES_TWO_LAYER) {
569 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
574 /* calculate required pbl page size */
575 pbl_size = MIN_FW_PBL_PAGE_SIZE;
576 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
577 NUM_PBES_ON_PAGE(pbl_size);
579 while (pbl_capacity < num_pbes) {
581 pbl_capacity = pbl_size / sizeof(u64);
582 pbl_capacity = pbl_capacity * pbl_capacity;
585 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
586 num_pbls++; /* One for the layer0 ( points to the pbls) */
587 pbl_info->two_layered = true;
589 /* One layered PBL */
591 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
592 roundup_pow_of_two((num_pbes * sizeof(u64))));
593 pbl_info->two_layered = false;
596 pbl_info->num_pbls = num_pbls;
597 pbl_info->pbl_size = pbl_size;
598 pbl_info->num_pbes = num_pbes;
600 DP_DEBUG(dev, QEDR_MSG_MR,
601 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
602 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
607 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
608 struct qedr_pbl *pbl,
609 struct qedr_pbl_info *pbl_info, u32 pg_shift)
611 int pbe_cnt, total_num_pbes = 0;
612 u32 fw_pg_cnt, fw_pg_per_umem_pg;
613 struct qedr_pbl *pbl_tbl;
614 struct sg_dma_page_iter sg_iter;
618 if (!pbl_info->num_pbes)
621 /* If we have a two layered pbl, the first pbl points to the rest
622 * of the pbls and the first entry lays on the second pbl in the table
624 if (pbl_info->two_layered)
629 pbe = (struct regpair *)pbl_tbl->va;
631 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
637 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
639 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
640 pg_addr = sg_page_iter_dma_address(&sg_iter);
641 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
642 pbe->lo = cpu_to_le32(pg_addr);
643 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
645 pg_addr += BIT(pg_shift);
650 if (total_num_pbes == pbl_info->num_pbes)
653 /* If the given pbl is full storing the pbes,
656 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
658 pbe = (struct regpair *)pbl_tbl->va;
667 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
668 struct qedr_cq *cq, struct ib_udata *udata)
670 struct qedr_create_cq_uresp uresp;
673 memset(&uresp, 0, sizeof(uresp));
675 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
676 uresp.icid = cq->icid;
678 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
680 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
685 static void consume_cqe(struct qedr_cq *cq)
687 if (cq->latest_cqe == cq->toggle_cqe)
688 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
690 cq->latest_cqe = qed_chain_consume(&cq->pbl);
693 static inline int qedr_align_cq_entries(int entries)
695 u64 size, aligned_size;
697 /* We allocate an extra entry that we don't report to the FW. */
698 size = (entries + 1) * QEDR_CQE_SIZE;
699 aligned_size = ALIGN(size, PAGE_SIZE);
701 return aligned_size / QEDR_CQE_SIZE;
704 static inline int qedr_init_user_queue(struct ib_udata *udata,
705 struct qedr_dev *dev,
706 struct qedr_userq *q, u64 buf_addr,
707 size_t buf_len, int access, int dmasync,
713 q->buf_addr = buf_addr;
714 q->buf_len = buf_len;
715 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
716 if (IS_ERR(q->umem)) {
717 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
719 return PTR_ERR(q->umem);
722 fw_pages = ib_umem_page_count(q->umem) <<
723 (PAGE_SHIFT - FW_PAGE_SHIFT);
725 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
729 if (alloc_and_init) {
730 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
731 if (IS_ERR(q->pbl_tbl)) {
732 rc = PTR_ERR(q->pbl_tbl);
735 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
738 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
748 ib_umem_release(q->umem);
754 static inline void qedr_init_cq_params(struct qedr_cq *cq,
755 struct qedr_ucontext *ctx,
756 struct qedr_dev *dev, int vector,
757 int chain_entries, int page_cnt,
759 struct qed_rdma_create_cq_in_params
762 memset(params, 0, sizeof(*params));
763 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
764 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
765 params->cnq_id = vector;
766 params->cq_size = chain_entries - 1;
767 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
768 params->pbl_num_pages = page_cnt;
769 params->pbl_ptr = pbl_ptr;
770 params->pbl_two_level = 0;
773 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
775 cq->db.data.agg_flags = flags;
776 cq->db.data.value = cpu_to_le32(cons);
777 writeq(cq->db.raw, cq->db_addr);
780 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
782 struct qedr_cq *cq = get_qedr_cq(ibcq);
783 unsigned long sflags;
784 struct qedr_dev *dev;
786 dev = get_qedr_dev(ibcq->device);
790 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
796 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
799 spin_lock_irqsave(&cq->cq_lock, sflags);
803 if (flags & IB_CQ_SOLICITED)
804 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
806 if (flags & IB_CQ_NEXT_COMP)
807 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
809 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
811 spin_unlock_irqrestore(&cq->cq_lock, sflags);
816 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
817 const struct ib_cq_init_attr *attr,
818 struct ib_udata *udata)
820 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
821 udata, struct qedr_ucontext, ibucontext);
822 struct qed_rdma_destroy_cq_out_params destroy_oparams;
823 struct qed_rdma_destroy_cq_in_params destroy_iparams;
824 struct qedr_dev *dev = get_qedr_dev(ibdev);
825 struct qed_rdma_create_cq_in_params params;
826 struct qedr_create_cq_ureq ureq;
827 int vector = attr->comp_vector;
828 int entries = attr->cqe;
836 DP_DEBUG(dev, QEDR_MSG_INIT,
837 "create_cq: called from %s. entries=%d, vector=%d\n",
838 udata ? "User Lib" : "Kernel", entries, vector);
840 if (entries > QEDR_MAX_CQES) {
842 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
843 entries, QEDR_MAX_CQES);
844 return ERR_PTR(-EINVAL);
847 chain_entries = qedr_align_cq_entries(entries);
848 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
850 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
852 return ERR_PTR(-ENOMEM);
855 memset(&ureq, 0, sizeof(ureq));
856 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
858 "create cq: problem copying data from user space\n");
864 "create cq: cannot create a cq with 0 entries\n");
868 cq->cq_type = QEDR_CQ_TYPE_USER;
870 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
871 ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
876 pbl_ptr = cq->q.pbl_tbl->pa;
877 page_cnt = cq->q.pbl_info.num_pbes;
879 cq->ibcq.cqe = chain_entries;
881 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
883 rc = dev->ops->common->chain_alloc(dev->cdev,
884 QED_CHAIN_USE_TO_CONSUME,
886 QED_CHAIN_CNT_TYPE_U32,
888 sizeof(union rdma_cqe),
893 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
894 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
895 cq->ibcq.cqe = cq->pbl.capacity;
898 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
901 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
906 cq->sig = QEDR_CQ_MAGIC_NUMBER;
907 spin_lock_init(&cq->cq_lock);
910 rc = qedr_copy_cq_uresp(dev, cq, udata);
914 /* Generate doorbell address. */
915 cq->db_addr = dev->db_addr +
916 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
917 cq->db.data.icid = cq->icid;
918 cq->db.data.params = DB_AGG_CMD_SET <<
919 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
921 /* point to the very last element, passing it we will toggle */
922 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
923 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
924 cq->latest_cqe = NULL;
926 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
929 DP_DEBUG(dev, QEDR_MSG_CQ,
930 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
931 cq->icid, cq, params.cq_size);
936 destroy_iparams.icid = cq->icid;
937 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
941 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
943 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
946 ib_umem_release(cq->q.umem);
949 return ERR_PTR(-EINVAL);
952 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
954 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
955 struct qedr_cq *cq = get_qedr_cq(ibcq);
957 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
962 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
963 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
965 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
967 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
968 struct qed_rdma_destroy_cq_out_params oparams;
969 struct qed_rdma_destroy_cq_in_params iparams;
970 struct qedr_cq *cq = get_qedr_cq(ibcq);
974 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
978 /* GSIs CQs are handled by driver, so they don't exist in the FW */
979 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
982 iparams.icid = cq->icid;
983 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
987 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
991 ib_umem_release(cq->q.umem);
994 /* We don't want the IRQ handler to handle a non-existing CQ so we
995 * wait until all CNQ interrupts, if any, are received. This will always
996 * happen and will always happen very fast. If not, then a serious error
997 * has occured. That is why we can use a long delay.
998 * We spin for a short time so we don’t lose time on context switching
999 * in case all the completions are handled in that span. Otherwise
1000 * we sleep for a while and check again. Since the CNQ may be
1001 * associated with (only) the current CPU we use msleep to allow the
1002 * current CPU to be freed.
1003 * The CNQ notification is increased in qedr_irq_handler().
1005 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1006 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1007 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1011 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1012 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1013 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1017 if (oparams.num_cq_notif != cq->cnq_notif)
1020 /* Note that we don't need to have explicit code to wait for the
1021 * completion of the event handler because it is invoked from the EQ.
1022 * Since the destroy CQ ramrod has also been received on the EQ we can
1023 * be certain that there's no event handler in process.
1034 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1035 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1040 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1041 struct ib_qp_attr *attr,
1043 struct qed_rdma_modify_qp_in_params
1046 const struct ib_gid_attr *gid_attr;
1047 enum rdma_network_type nw_type;
1048 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1053 gid_attr = grh->sgid_attr;
1054 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1058 nw_type = rdma_gid_attr_network_type(gid_attr);
1060 case RDMA_NETWORK_IPV6:
1061 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1062 sizeof(qp_params->sgid));
1063 memcpy(&qp_params->dgid.bytes[0],
1065 sizeof(qp_params->dgid));
1066 qp_params->roce_mode = ROCE_V2_IPV6;
1067 SET_FIELD(qp_params->modify_flags,
1068 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1070 case RDMA_NETWORK_IB:
1071 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1072 sizeof(qp_params->sgid));
1073 memcpy(&qp_params->dgid.bytes[0],
1075 sizeof(qp_params->dgid));
1076 qp_params->roce_mode = ROCE_V1;
1078 case RDMA_NETWORK_IPV4:
1079 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1080 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1081 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1082 qp_params->sgid.ipv4_addr = ipv4_addr;
1084 qedr_get_ipv4_from_gid(grh->dgid.raw);
1085 qp_params->dgid.ipv4_addr = ipv4_addr;
1086 SET_FIELD(qp_params->modify_flags,
1087 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1088 qp_params->roce_mode = ROCE_V2_IPV4;
1092 for (i = 0; i < 4; i++) {
1093 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1094 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1097 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1098 qp_params->vlan_id = 0;
1103 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1104 struct ib_qp_init_attr *attrs,
1105 struct ib_udata *udata)
1107 struct qedr_device_attr *qattr = &dev->attr;
1109 /* QP0... attrs->qp_type == IB_QPT_GSI */
1110 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1111 DP_DEBUG(dev, QEDR_MSG_QP,
1112 "create qp: unsupported qp type=0x%x requested\n",
1117 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1119 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1120 attrs->cap.max_send_wr, qattr->max_sqe);
1124 if (attrs->cap.max_inline_data > qattr->max_inline) {
1126 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1127 attrs->cap.max_inline_data, qattr->max_inline);
1131 if (attrs->cap.max_send_sge > qattr->max_sge) {
1133 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1134 attrs->cap.max_send_sge, qattr->max_sge);
1138 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1140 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1141 attrs->cap.max_recv_sge, qattr->max_sge);
1145 /* Unprivileged user space cannot create special QP */
1146 if (udata && attrs->qp_type == IB_QPT_GSI) {
1148 "create qp: userspace can't create special QPs of type=0x%x\n",
1156 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1157 struct qedr_srq *srq, struct ib_udata *udata)
1159 struct qedr_create_srq_uresp uresp = {};
1162 uresp.srq_id = srq->srq_id;
1164 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1166 DP_ERR(dev, "create srq: problem copying data to user space\n");
1171 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1172 struct qedr_create_qp_uresp *uresp,
1175 /* iWARP requires two doorbells per RQ. */
1176 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1177 uresp->rq_db_offset =
1178 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1179 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1181 uresp->rq_db_offset =
1182 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1185 uresp->rq_icid = qp->icid;
1188 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1189 struct qedr_create_qp_uresp *uresp,
1192 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1194 /* iWARP uses the same cid for rq and sq */
1195 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1196 uresp->sq_icid = qp->icid;
1198 uresp->sq_icid = qp->icid + 1;
1201 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1202 struct qedr_qp *qp, struct ib_udata *udata)
1204 struct qedr_create_qp_uresp uresp;
1207 memset(&uresp, 0, sizeof(uresp));
1208 qedr_copy_sq_uresp(dev, &uresp, qp);
1209 qedr_copy_rq_uresp(dev, &uresp, qp);
1211 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1212 uresp.qp_id = qp->qp_id;
1214 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1217 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1223 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1226 struct ib_qp_init_attr *attrs)
1228 spin_lock_init(&qp->q_lock);
1229 atomic_set(&qp->refcnt, 1);
1231 qp->qp_type = attrs->qp_type;
1232 qp->max_inline_data = attrs->cap.max_inline_data;
1233 qp->sq.max_sges = attrs->cap.max_send_sge;
1234 qp->state = QED_ROCE_QP_STATE_RESET;
1235 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1236 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1240 qp->srq = get_qedr_srq(attrs->srq);
1242 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1243 qp->rq.max_sges = attrs->cap.max_recv_sge;
1244 DP_DEBUG(dev, QEDR_MSG_QP,
1245 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1246 qp->rq.max_sges, qp->rq_cq->icid);
1249 DP_DEBUG(dev, QEDR_MSG_QP,
1250 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1251 pd->pd_id, qp->qp_type, qp->max_inline_data,
1252 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1253 DP_DEBUG(dev, QEDR_MSG_QP,
1254 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1255 qp->sq.max_sges, qp->sq_cq->icid);
1258 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1260 qp->sq.db = dev->db_addr +
1261 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1262 qp->sq.db_data.data.icid = qp->icid + 1;
1264 qp->rq.db = dev->db_addr +
1265 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1266 qp->rq.db_data.data.icid = qp->icid;
1270 static int qedr_check_srq_params(struct qedr_dev *dev,
1271 struct ib_srq_init_attr *attrs,
1272 struct ib_udata *udata)
1274 struct qedr_device_attr *qattr = &dev->attr;
1276 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1278 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1279 attrs->attr.max_wr, qattr->max_srq_wr);
1283 if (attrs->attr.max_sge > qattr->max_sge) {
1285 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1286 attrs->attr.max_sge, qattr->max_sge);
1293 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1295 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1296 ib_umem_release(srq->usrq.umem);
1297 ib_umem_release(srq->prod_umem);
1300 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1302 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1303 struct qedr_dev *dev = srq->dev;
1305 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1307 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1308 hw_srq->virt_prod_pair_addr,
1309 hw_srq->phy_prod_pair_addr);
1312 static int qedr_init_srq_user_params(struct ib_udata *udata,
1313 struct qedr_srq *srq,
1314 struct qedr_create_srq_ureq *ureq,
1315 int access, int dmasync)
1317 struct scatterlist *sg;
1320 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1321 ureq->srq_len, access, dmasync, 1);
1326 ib_umem_get(udata, ureq->prod_pair_addr,
1327 sizeof(struct rdma_srq_producers), access, dmasync);
1328 if (IS_ERR(srq->prod_umem)) {
1329 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1330 ib_umem_release(srq->usrq.umem);
1332 "create srq: failed ib_umem_get for producer, got %ld\n",
1333 PTR_ERR(srq->prod_umem));
1334 return PTR_ERR(srq->prod_umem);
1337 sg = srq->prod_umem->sg_head.sgl;
1338 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1343 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1344 struct qedr_dev *dev,
1345 struct ib_srq_init_attr *init_attr)
1347 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1348 dma_addr_t phy_prod_pair_addr;
1353 va = dma_alloc_coherent(&dev->pdev->dev,
1354 sizeof(struct rdma_srq_producers),
1355 &phy_prod_pair_addr, GFP_KERNEL);
1358 "create srq: failed to allocate dma memory for producer\n");
1362 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1363 hw_srq->virt_prod_pair_addr = va;
1365 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1366 rc = dev->ops->common->chain_alloc(dev->cdev,
1367 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1369 QED_CHAIN_CNT_TYPE_U32,
1371 QEDR_SRQ_WQE_ELEM_SIZE,
1372 &hw_srq->pbl, NULL);
1376 hw_srq->num_elems = num_elems;
1381 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1382 va, phy_prod_pair_addr);
1386 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1387 struct ib_udata *udata)
1389 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1390 struct qed_rdma_create_srq_in_params in_params = {};
1391 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1392 struct qed_rdma_create_srq_out_params out_params;
1393 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1394 struct qedr_create_srq_ureq ureq = {};
1395 u64 pbl_base_addr, phy_prod_pair_addr;
1396 struct qedr_srq_hwq_info *hw_srq;
1397 u32 page_cnt, page_size;
1398 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1401 DP_DEBUG(dev, QEDR_MSG_QP,
1402 "create SRQ called from %s (pd %p)\n",
1403 (udata) ? "User lib" : "kernel", pd);
1405 rc = qedr_check_srq_params(dev, init_attr, udata);
1410 hw_srq = &srq->hw_srq;
1411 spin_lock_init(&srq->lock);
1413 hw_srq->max_wr = init_attr->attr.max_wr;
1414 hw_srq->max_sges = init_attr->attr.max_sge;
1417 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1419 "create srq: problem copying data from user space\n");
1423 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
1427 page_cnt = srq->usrq.pbl_info.num_pbes;
1428 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1429 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1430 page_size = PAGE_SIZE;
1432 struct qed_chain *pbl;
1434 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1439 page_cnt = qed_chain_get_page_cnt(pbl);
1440 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1441 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1442 page_size = QED_CHAIN_PAGE_SIZE;
1445 in_params.pd_id = pd->pd_id;
1446 in_params.pbl_base_addr = pbl_base_addr;
1447 in_params.prod_pair_addr = phy_prod_pair_addr;
1448 in_params.num_pages = page_cnt;
1449 in_params.page_size = page_size;
1451 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1455 srq->srq_id = out_params.srq_id;
1458 rc = qedr_copy_srq_uresp(dev, srq, udata);
1463 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1467 DP_DEBUG(dev, QEDR_MSG_SRQ,
1468 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1472 destroy_in_params.srq_id = srq->srq_id;
1474 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1477 qedr_free_srq_user_params(srq);
1479 qedr_free_srq_kernel_params(srq);
1484 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1486 struct qed_rdma_destroy_srq_in_params in_params = {};
1487 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1488 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1490 xa_erase_irq(&dev->srqs, srq->srq_id);
1491 in_params.srq_id = srq->srq_id;
1492 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1495 qedr_free_srq_user_params(srq);
1497 qedr_free_srq_kernel_params(srq);
1499 DP_DEBUG(dev, QEDR_MSG_SRQ,
1500 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1504 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1505 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1507 struct qed_rdma_modify_srq_in_params in_params = {};
1508 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1509 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1512 if (attr_mask & IB_SRQ_MAX_WR) {
1514 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1519 if (attr_mask & IB_SRQ_LIMIT) {
1520 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1522 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1523 attr->srq_limit, srq->hw_srq.max_wr);
1527 in_params.srq_id = srq->srq_id;
1528 in_params.wqe_limit = attr->srq_limit;
1529 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1534 srq->srq_limit = attr->srq_limit;
1536 DP_DEBUG(dev, QEDR_MSG_SRQ,
1537 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1543 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1546 struct ib_qp_init_attr *attrs,
1547 bool fmr_and_reserved_lkey,
1548 struct qed_rdma_create_qp_in_params *params)
1550 /* QP handle to be written in an async event */
1551 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1552 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1554 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1555 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1556 params->pd = pd->pd_id;
1557 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1558 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1559 params->stats_queue = 0;
1561 params->use_srq = false;
1564 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1567 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1568 params->srq_id = qp->srq->srq_id;
1569 params->use_srq = true;
1573 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1575 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1584 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1588 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1590 struct qed_rdma_create_qp_out_params *out_params)
1592 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1593 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1595 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1596 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1598 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1599 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1602 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1603 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1606 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1609 ib_umem_release(qp->usq.umem);
1610 qp->usq.umem = NULL;
1613 ib_umem_release(qp->urq.umem);
1614 qp->urq.umem = NULL;
1617 static int qedr_create_user_qp(struct qedr_dev *dev,
1620 struct ib_udata *udata,
1621 struct ib_qp_init_attr *attrs)
1623 struct qed_rdma_create_qp_in_params in_params;
1624 struct qed_rdma_create_qp_out_params out_params;
1625 struct qedr_pd *pd = get_qedr_pd(ibpd);
1626 struct qedr_create_qp_ureq ureq;
1627 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1630 memset(&ureq, 0, sizeof(ureq));
1631 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1633 DP_ERR(dev, "Problem copying data from user space\n");
1637 /* SQ - read access only (0), dma sync not required (0) */
1638 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1639 ureq.sq_len, 0, 0, alloc_and_init);
1644 /* RQ - read access only (0), dma sync not required (0) */
1645 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1646 ureq.rq_len, 0, 0, alloc_and_init);
1651 memset(&in_params, 0, sizeof(in_params));
1652 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1653 in_params.qp_handle_lo = ureq.qp_handle_lo;
1654 in_params.qp_handle_hi = ureq.qp_handle_hi;
1655 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1656 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1658 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1659 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1662 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1663 &in_params, &out_params);
1670 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1671 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1673 qp->qp_id = out_params.qp_id;
1674 qp->icid = out_params.icid;
1676 rc = qedr_copy_qp_uresp(dev, qp, udata);
1680 qedr_qp_user_print(dev, qp);
1684 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1686 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1689 qedr_cleanup_user(dev, qp);
1693 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1695 qp->sq.db = dev->db_addr +
1696 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1697 qp->sq.db_data.data.icid = qp->icid;
1699 qp->rq.db = dev->db_addr +
1700 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1701 qp->rq.db_data.data.icid = qp->icid;
1702 qp->rq.iwarp_db2 = dev->db_addr +
1703 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1704 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1705 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1709 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1711 struct qed_rdma_create_qp_in_params *in_params,
1712 u32 n_sq_elems, u32 n_rq_elems)
1714 struct qed_rdma_create_qp_out_params out_params;
1717 rc = dev->ops->common->chain_alloc(dev->cdev,
1718 QED_CHAIN_USE_TO_PRODUCE,
1720 QED_CHAIN_CNT_TYPE_U32,
1722 QEDR_SQE_ELEMENT_SIZE,
1728 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1729 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1731 rc = dev->ops->common->chain_alloc(dev->cdev,
1732 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1734 QED_CHAIN_CNT_TYPE_U32,
1736 QEDR_RQE_ELEMENT_SIZE,
1741 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1742 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1744 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1745 in_params, &out_params);
1750 qp->qp_id = out_params.qp_id;
1751 qp->icid = out_params.icid;
1753 qedr_set_roce_db_info(dev, qp);
1758 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1760 struct qed_rdma_create_qp_in_params *in_params,
1761 u32 n_sq_elems, u32 n_rq_elems)
1763 struct qed_rdma_create_qp_out_params out_params;
1764 struct qed_chain_ext_pbl ext_pbl;
1767 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1768 QEDR_SQE_ELEMENT_SIZE,
1769 QED_CHAIN_MODE_PBL);
1770 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1771 QEDR_RQE_ELEMENT_SIZE,
1772 QED_CHAIN_MODE_PBL);
1774 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1775 in_params, &out_params);
1780 /* Now we allocate the chain */
1781 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1782 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1784 rc = dev->ops->common->chain_alloc(dev->cdev,
1785 QED_CHAIN_USE_TO_PRODUCE,
1787 QED_CHAIN_CNT_TYPE_U32,
1789 QEDR_SQE_ELEMENT_SIZE,
1790 &qp->sq.pbl, &ext_pbl);
1795 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1796 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1798 rc = dev->ops->common->chain_alloc(dev->cdev,
1799 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1801 QED_CHAIN_CNT_TYPE_U32,
1803 QEDR_RQE_ELEMENT_SIZE,
1804 &qp->rq.pbl, &ext_pbl);
1809 qp->qp_id = out_params.qp_id;
1810 qp->icid = out_params.icid;
1812 qedr_set_iwarp_db_info(dev, qp);
1816 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1821 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1823 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1824 kfree(qp->wqe_wr_id);
1826 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1827 kfree(qp->rqe_wr_id);
1830 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1833 struct ib_qp_init_attr *attrs)
1835 struct qed_rdma_create_qp_in_params in_params;
1836 struct qedr_pd *pd = get_qedr_pd(ibpd);
1842 memset(&in_params, 0, sizeof(in_params));
1844 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1845 * the ring. The ring should allow at least a single WR, even if the
1846 * user requested none, due to allocation issues.
1847 * We should add an extra WR since the prod and cons indices of
1848 * wqe_wr_id are managed in such a way that the WQ is considered full
1849 * when (prod+1)%max_wr==cons. We currently don't do that because we
1850 * double the number of entries due an iSER issue that pushes far more
1851 * WRs than indicated. If we decline its ib_post_send() then we get
1852 * error prints in the dmesg we'd like to avoid.
1854 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1857 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1859 if (!qp->wqe_wr_id) {
1860 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1864 /* QP handle to be written in CQE */
1865 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1866 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1868 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1869 * the ring. There ring should allow at least a single WR, even if the
1870 * user requested none, due to allocation issues.
1872 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1874 /* Allocate driver internal RQ array */
1875 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1877 if (!qp->rqe_wr_id) {
1879 "create qp: failed RQ shadow memory allocation\n");
1880 kfree(qp->wqe_wr_id);
1884 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1886 n_sq_entries = attrs->cap.max_send_wr;
1887 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1888 n_sq_entries = max_t(u32, n_sq_entries, 1);
1889 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1891 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1893 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1894 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1895 n_sq_elems, n_rq_elems);
1897 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1898 n_sq_elems, n_rq_elems);
1900 qedr_cleanup_kernel(dev, qp);
1905 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1906 struct ib_qp_init_attr *attrs,
1907 struct ib_udata *udata)
1909 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1910 struct qedr_pd *pd = get_qedr_pd(ibpd);
1915 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1916 udata ? "user library" : "kernel", pd);
1918 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
1922 DP_DEBUG(dev, QEDR_MSG_QP,
1923 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1924 udata ? "user library" : "kernel", attrs->event_handler, pd,
1925 get_qedr_cq(attrs->send_cq),
1926 get_qedr_cq(attrs->send_cq)->icid,
1927 get_qedr_cq(attrs->recv_cq),
1928 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
1930 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1932 DP_ERR(dev, "create qp: failed allocating memory\n");
1933 return ERR_PTR(-ENOMEM);
1936 qedr_set_common_qp_params(dev, qp, pd, attrs);
1938 if (attrs->qp_type == IB_QPT_GSI) {
1939 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1946 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1948 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1953 qp->ibqp.qp_num = qp->qp_id;
1955 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1956 rc = xa_insert_irq(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1966 return ERR_PTR(-EFAULT);
1969 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1972 case QED_ROCE_QP_STATE_RESET:
1973 return IB_QPS_RESET;
1974 case QED_ROCE_QP_STATE_INIT:
1976 case QED_ROCE_QP_STATE_RTR:
1978 case QED_ROCE_QP_STATE_RTS:
1980 case QED_ROCE_QP_STATE_SQD:
1982 case QED_ROCE_QP_STATE_ERR:
1984 case QED_ROCE_QP_STATE_SQE:
1990 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1991 enum ib_qp_state qp_state)
1995 return QED_ROCE_QP_STATE_RESET;
1997 return QED_ROCE_QP_STATE_INIT;
1999 return QED_ROCE_QP_STATE_RTR;
2001 return QED_ROCE_QP_STATE_RTS;
2003 return QED_ROCE_QP_STATE_SQD;
2005 return QED_ROCE_QP_STATE_ERR;
2007 return QED_ROCE_QP_STATE_ERR;
2011 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2013 qed_chain_reset(&qph->pbl);
2017 qph->db_data.data.value = cpu_to_le16(0);
2020 static int qedr_update_qp_state(struct qedr_dev *dev,
2022 enum qed_roce_qp_state cur_state,
2023 enum qed_roce_qp_state new_state)
2027 if (new_state == cur_state)
2030 switch (cur_state) {
2031 case QED_ROCE_QP_STATE_RESET:
2032 switch (new_state) {
2033 case QED_ROCE_QP_STATE_INIT:
2034 qp->prev_wqe_size = 0;
2035 qedr_reset_qp_hwq_info(&qp->sq);
2036 qedr_reset_qp_hwq_info(&qp->rq);
2043 case QED_ROCE_QP_STATE_INIT:
2044 switch (new_state) {
2045 case QED_ROCE_QP_STATE_RTR:
2046 /* Update doorbell (in case post_recv was
2047 * done before move to RTR)
2050 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2051 writel(qp->rq.db_data.raw, qp->rq.db);
2054 case QED_ROCE_QP_STATE_ERR:
2057 /* Invalid state change. */
2062 case QED_ROCE_QP_STATE_RTR:
2064 switch (new_state) {
2065 case QED_ROCE_QP_STATE_RTS:
2067 case QED_ROCE_QP_STATE_ERR:
2070 /* Invalid state change. */
2075 case QED_ROCE_QP_STATE_RTS:
2077 switch (new_state) {
2078 case QED_ROCE_QP_STATE_SQD:
2080 case QED_ROCE_QP_STATE_ERR:
2083 /* Invalid state change. */
2088 case QED_ROCE_QP_STATE_SQD:
2090 switch (new_state) {
2091 case QED_ROCE_QP_STATE_RTS:
2092 case QED_ROCE_QP_STATE_ERR:
2095 /* Invalid state change. */
2100 case QED_ROCE_QP_STATE_ERR:
2102 switch (new_state) {
2103 case QED_ROCE_QP_STATE_RESET:
2104 if ((qp->rq.prod != qp->rq.cons) ||
2105 (qp->sq.prod != qp->sq.cons)) {
2107 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2108 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2126 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2127 int attr_mask, struct ib_udata *udata)
2129 struct qedr_qp *qp = get_qedr_qp(ibqp);
2130 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2131 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2132 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2133 enum ib_qp_state old_qp_state, new_qp_state;
2134 enum qed_roce_qp_state cur_state;
2137 DP_DEBUG(dev, QEDR_MSG_QP,
2138 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2141 old_qp_state = qedr_get_ibqp_state(qp->state);
2142 if (attr_mask & IB_QP_STATE)
2143 new_qp_state = attr->qp_state;
2145 new_qp_state = old_qp_state;
2147 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2148 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2149 ibqp->qp_type, attr_mask)) {
2151 "modify qp: invalid attribute mask=0x%x specified for\n"
2152 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2153 attr_mask, qp->qp_id, ibqp->qp_type,
2154 old_qp_state, new_qp_state);
2160 /* Translate the masks... */
2161 if (attr_mask & IB_QP_STATE) {
2162 SET_FIELD(qp_params.modify_flags,
2163 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2164 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2167 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2168 qp_params.sqd_async = true;
2170 if (attr_mask & IB_QP_PKEY_INDEX) {
2171 SET_FIELD(qp_params.modify_flags,
2172 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2173 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2178 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2181 if (attr_mask & IB_QP_QKEY)
2182 qp->qkey = attr->qkey;
2184 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2185 SET_FIELD(qp_params.modify_flags,
2186 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2187 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2188 IB_ACCESS_REMOTE_READ;
2189 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2190 IB_ACCESS_REMOTE_WRITE;
2191 qp_params.incoming_atomic_en = attr->qp_access_flags &
2192 IB_ACCESS_REMOTE_ATOMIC;
2195 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2196 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2199 if (attr_mask & IB_QP_PATH_MTU) {
2200 if (attr->path_mtu < IB_MTU_256 ||
2201 attr->path_mtu > IB_MTU_4096) {
2202 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2206 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2207 ib_mtu_enum_to_int(iboe_get_mtu
2213 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2214 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2217 SET_FIELD(qp_params.modify_flags,
2218 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2220 qp_params.traffic_class_tos = grh->traffic_class;
2221 qp_params.flow_label = grh->flow_label;
2222 qp_params.hop_limit_ttl = grh->hop_limit;
2224 qp->sgid_idx = grh->sgid_index;
2226 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2229 "modify qp: problems with GID index %d (rc=%d)\n",
2230 grh->sgid_index, rc);
2234 rc = qedr_get_dmac(dev, &attr->ah_attr,
2235 qp_params.remote_mac_addr);
2239 qp_params.use_local_mac = true;
2240 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2242 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2243 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2244 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2245 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2246 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2247 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2248 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2249 qp_params.remote_mac_addr);
2251 qp_params.mtu = qp->mtu;
2252 qp_params.lb_indication = false;
2255 if (!qp_params.mtu) {
2256 /* Stay with current MTU */
2258 qp_params.mtu = qp->mtu;
2261 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2264 if (attr_mask & IB_QP_TIMEOUT) {
2265 SET_FIELD(qp_params.modify_flags,
2266 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2268 /* The received timeout value is an exponent used like this:
2269 * "12.7.34 LOCAL ACK TIMEOUT
2270 * Value representing the transport (ACK) timeout for use by
2271 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2272 * The FW expects timeout in msec so we need to divide the usec
2273 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2274 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2275 * The value of zero means infinite so we use a 'max_t' to make
2276 * sure that sub 1 msec values will be configured as 1 msec.
2279 qp_params.ack_timeout =
2280 1 << max_t(int, attr->timeout - 8, 0);
2282 qp_params.ack_timeout = 0;
2285 if (attr_mask & IB_QP_RETRY_CNT) {
2286 SET_FIELD(qp_params.modify_flags,
2287 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2288 qp_params.retry_cnt = attr->retry_cnt;
2291 if (attr_mask & IB_QP_RNR_RETRY) {
2292 SET_FIELD(qp_params.modify_flags,
2293 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2294 qp_params.rnr_retry_cnt = attr->rnr_retry;
2297 if (attr_mask & IB_QP_RQ_PSN) {
2298 SET_FIELD(qp_params.modify_flags,
2299 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2300 qp_params.rq_psn = attr->rq_psn;
2301 qp->rq_psn = attr->rq_psn;
2304 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2305 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2308 "unsupported max_rd_atomic=%d, supported=%d\n",
2309 attr->max_rd_atomic,
2310 dev->attr.max_qp_req_rd_atomic_resc);
2314 SET_FIELD(qp_params.modify_flags,
2315 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2316 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2319 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2320 SET_FIELD(qp_params.modify_flags,
2321 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2322 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2325 if (attr_mask & IB_QP_SQ_PSN) {
2326 SET_FIELD(qp_params.modify_flags,
2327 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2328 qp_params.sq_psn = attr->sq_psn;
2329 qp->sq_psn = attr->sq_psn;
2332 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2333 if (attr->max_dest_rd_atomic >
2334 dev->attr.max_qp_resp_rd_atomic_resc) {
2336 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2337 attr->max_dest_rd_atomic,
2338 dev->attr.max_qp_resp_rd_atomic_resc);
2344 SET_FIELD(qp_params.modify_flags,
2345 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2346 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2349 if (attr_mask & IB_QP_DEST_QPN) {
2350 SET_FIELD(qp_params.modify_flags,
2351 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2353 qp_params.dest_qp = attr->dest_qp_num;
2354 qp->dest_qp_num = attr->dest_qp_num;
2357 cur_state = qp->state;
2359 /* Update the QP state before the actual ramrod to prevent a race with
2360 * fast path. Modifying the QP state to error will cause the device to
2361 * flush the CQEs and while polling the flushed CQEs will considered as
2362 * a potential issue if the QP isn't in error state.
2364 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2365 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2366 qp->state = QED_ROCE_QP_STATE_ERR;
2368 if (qp->qp_type != IB_QPT_GSI)
2369 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2370 qp->qed_qp, &qp_params);
2372 if (attr_mask & IB_QP_STATE) {
2373 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2374 rc = qedr_update_qp_state(dev, qp, cur_state,
2375 qp_params.new_state);
2376 qp->state = qp_params.new_state;
2383 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2385 int ib_qp_acc_flags = 0;
2387 if (params->incoming_rdma_write_en)
2388 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2389 if (params->incoming_rdma_read_en)
2390 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2391 if (params->incoming_atomic_en)
2392 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2393 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2394 return ib_qp_acc_flags;
2397 int qedr_query_qp(struct ib_qp *ibqp,
2398 struct ib_qp_attr *qp_attr,
2399 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2401 struct qed_rdma_query_qp_out_params params;
2402 struct qedr_qp *qp = get_qedr_qp(ibqp);
2403 struct qedr_dev *dev = qp->dev;
2406 memset(¶ms, 0, sizeof(params));
2408 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2412 memset(qp_attr, 0, sizeof(*qp_attr));
2413 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2415 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2416 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2417 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2418 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2419 qp_attr->rq_psn = params.rq_psn;
2420 qp_attr->sq_psn = params.sq_psn;
2421 qp_attr->dest_qp_num = params.dest_qp;
2423 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2425 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2426 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2427 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2428 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2429 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2430 qp_init_attr->cap = qp_attr->cap;
2432 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2433 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2434 params.flow_label, qp->sgid_idx,
2435 params.hop_limit_ttl, params.traffic_class_tos);
2436 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2437 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2438 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2439 qp_attr->timeout = params.timeout;
2440 qp_attr->rnr_retry = params.rnr_retry;
2441 qp_attr->retry_cnt = params.retry_cnt;
2442 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2443 qp_attr->pkey_index = params.pkey_index;
2444 qp_attr->port_num = 1;
2445 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2446 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2447 qp_attr->alt_pkey_index = 0;
2448 qp_attr->alt_port_num = 0;
2449 qp_attr->alt_timeout = 0;
2450 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2452 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2453 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2454 qp_attr->max_rd_atomic = params.max_rd_atomic;
2455 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2457 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2458 qp_attr->cap.max_inline_data);
2464 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2465 struct ib_udata *udata)
2469 if (qp->qp_type != IB_QPT_GSI) {
2470 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2476 qedr_cleanup_user(dev, qp);
2478 qedr_cleanup_kernel(dev, qp);
2483 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2485 struct qedr_qp *qp = get_qedr_qp(ibqp);
2486 struct qedr_dev *dev = qp->dev;
2487 struct ib_qp_attr attr;
2491 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2494 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2495 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2496 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2497 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2499 attr.qp_state = IB_QPS_ERR;
2500 attr_mask |= IB_QP_STATE;
2502 /* Change the QP state to ERROR */
2503 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2506 /* Wait for the connect/accept to complete */
2510 while (qp->ep->during_connect) {
2511 DP_DEBUG(dev, QEDR_MSG_QP,
2512 "Still in during connect/accept\n");
2515 if (wait_count++ > 200) {
2517 "during connect timeout\n");
2524 if (qp->qp_type == IB_QPT_GSI)
2525 qedr_destroy_gsi_qp(dev);
2527 qedr_free_qp_resources(dev, qp, udata);
2529 if (atomic_dec_and_test(&qp->refcnt) &&
2530 rdma_protocol_iwarp(&dev->ibdev, 1)) {
2531 xa_erase_irq(&dev->qps, qp->qp_id);
2537 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
2538 struct ib_udata *udata)
2540 struct qedr_ah *ah = get_qedr_ah(ibah);
2542 rdma_copy_ah_attr(&ah->attr, attr);
2547 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2549 struct qedr_ah *ah = get_qedr_ah(ibah);
2551 rdma_destroy_ah_attr(&ah->attr);
2554 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2556 struct qedr_pbl *pbl, *tmp;
2558 if (info->pbl_table)
2559 list_add_tail(&info->pbl_table->list_entry,
2560 &info->free_pbl_list);
2562 if (!list_empty(&info->inuse_pbl_list))
2563 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2565 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2566 list_del(&pbl->list_entry);
2567 qedr_free_pbl(dev, &info->pbl_info, pbl);
2571 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2572 size_t page_list_len, bool two_layered)
2574 struct qedr_pbl *tmp;
2577 INIT_LIST_HEAD(&info->free_pbl_list);
2578 INIT_LIST_HEAD(&info->inuse_pbl_list);
2580 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2581 page_list_len, two_layered);
2585 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2586 if (IS_ERR(info->pbl_table)) {
2587 rc = PTR_ERR(info->pbl_table);
2591 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2592 &info->pbl_table->pa);
2594 /* in usual case we use 2 PBLs, so we add one to free
2595 * list and allocating another one
2597 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2599 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2603 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2605 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2609 free_mr_info(dev, info);
2614 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2615 u64 usr_addr, int acc, struct ib_udata *udata)
2617 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2622 pd = get_qedr_pd(ibpd);
2623 DP_DEBUG(dev, QEDR_MSG_MR,
2624 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2625 pd->pd_id, start, len, usr_addr, acc);
2627 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2628 return ERR_PTR(-EINVAL);
2630 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2634 mr->type = QEDR_MR_USER;
2636 mr->umem = ib_umem_get(udata, start, len, acc, 0);
2637 if (IS_ERR(mr->umem)) {
2642 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2646 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2647 &mr->info.pbl_info, PAGE_SHIFT);
2649 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2651 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2655 /* Index only, 18 bit long, lkey = itid << 8 | key */
2656 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2658 mr->hw_mr.pd = pd->pd_id;
2659 mr->hw_mr.local_read = 1;
2660 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2661 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2662 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2663 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2664 mr->hw_mr.mw_bind = false;
2665 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2666 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2667 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2668 mr->hw_mr.page_size_log = PAGE_SHIFT;
2669 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2670 mr->hw_mr.length = len;
2671 mr->hw_mr.vaddr = usr_addr;
2672 mr->hw_mr.zbva = false;
2673 mr->hw_mr.phy_mr = false;
2674 mr->hw_mr.dma_mr = false;
2676 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2678 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2682 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2683 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2684 mr->hw_mr.remote_atomic)
2685 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2687 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2692 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2694 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2700 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2702 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2703 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2706 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2710 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2712 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2713 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2715 /* it could be user registered memory. */
2717 ib_umem_release(mr->umem);
2724 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2725 int max_page_list_len)
2727 struct qedr_pd *pd = get_qedr_pd(ibpd);
2728 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2732 DP_DEBUG(dev, QEDR_MSG_MR,
2733 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2736 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2741 mr->type = QEDR_MR_FRMR;
2743 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2747 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2749 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2753 /* Index only, 18 bit long, lkey = itid << 8 | key */
2754 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2756 mr->hw_mr.pd = pd->pd_id;
2757 mr->hw_mr.local_read = 1;
2758 mr->hw_mr.local_write = 0;
2759 mr->hw_mr.remote_read = 0;
2760 mr->hw_mr.remote_write = 0;
2761 mr->hw_mr.remote_atomic = 0;
2762 mr->hw_mr.mw_bind = false;
2763 mr->hw_mr.pbl_ptr = 0;
2764 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2765 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2767 mr->hw_mr.length = 0;
2768 mr->hw_mr.vaddr = 0;
2769 mr->hw_mr.zbva = false;
2770 mr->hw_mr.phy_mr = true;
2771 mr->hw_mr.dma_mr = false;
2773 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2775 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2779 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2780 mr->ibmr.rkey = mr->ibmr.lkey;
2782 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2786 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2792 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2793 u32 max_num_sg, struct ib_udata *udata)
2797 if (mr_type != IB_MR_TYPE_MEM_REG)
2798 return ERR_PTR(-EINVAL);
2800 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2803 return ERR_PTR(-EINVAL);
2808 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2810 struct qedr_mr *mr = get_qedr_mr(ibmr);
2811 struct qedr_pbl *pbl_table;
2812 struct regpair *pbe;
2815 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2816 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2820 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2823 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2824 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2825 pbe = (struct regpair *)pbl_table->va;
2826 pbe += mr->npages % pbes_in_page;
2827 pbe->lo = cpu_to_le32((u32)addr);
2828 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2835 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2837 int work = info->completed - info->completed_handled - 1;
2839 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2840 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2841 struct qedr_pbl *pbl;
2843 /* Free all the page list that are possible to be freed
2844 * (all the ones that were invalidated), under the assumption
2845 * that if an FMR was completed successfully that means that
2846 * if there was an invalidate operation before it also ended
2848 pbl = list_first_entry(&info->inuse_pbl_list,
2849 struct qedr_pbl, list_entry);
2850 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2851 info->completed_handled++;
2855 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2856 int sg_nents, unsigned int *sg_offset)
2858 struct qedr_mr *mr = get_qedr_mr(ibmr);
2862 handle_completed_mrs(mr->dev, &mr->info);
2863 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2866 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2868 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2869 struct qedr_pd *pd = get_qedr_pd(ibpd);
2873 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2875 return ERR_PTR(-ENOMEM);
2877 mr->type = QEDR_MR_DMA;
2879 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2881 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2885 /* index only, 18 bit long, lkey = itid << 8 | key */
2886 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2887 mr->hw_mr.pd = pd->pd_id;
2888 mr->hw_mr.local_read = 1;
2889 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2890 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2891 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2892 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2893 mr->hw_mr.dma_mr = true;
2895 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2897 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2901 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2902 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2903 mr->hw_mr.remote_atomic)
2904 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2906 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2910 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2916 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2918 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2921 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2925 for (i = 0; i < num_sge; i++)
2926 len += sg_list[i].length;
2931 static void swap_wqe_data64(u64 *p)
2935 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2936 *p = cpu_to_be64(cpu_to_le64(*p));
2939 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2940 struct qedr_qp *qp, u8 *wqe_size,
2941 const struct ib_send_wr *wr,
2942 const struct ib_send_wr **bad_wr,
2945 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2946 char *seg_prt, *wqe;
2949 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2950 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2964 /* Copy data inline */
2965 for (i = 0; i < wr->num_sge; i++) {
2966 u32 len = wr->sg_list[i].length;
2967 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2972 /* New segment required */
2974 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2976 seg_siz = sizeof(struct rdma_sq_common_wqe);
2980 /* Calculate currently allowed length */
2981 cur = min_t(u32, len, seg_siz);
2982 memcpy(seg_prt, src, cur);
2984 /* Update segment variables */
2988 /* Update sge variables */
2992 /* Swap fully-completed segments */
2994 swap_wqe_data64((u64 *)wqe);
2998 /* swap last not completed segment */
3000 swap_wqe_data64((u64 *)wqe);
3005 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3007 DMA_REGPAIR_LE(sge->addr, vaddr); \
3008 (sge)->length = cpu_to_le32(vlength); \
3009 (sge)->flags = cpu_to_le32(vflags); \
3012 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3014 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3015 (hdr)->num_sges = num_sge; \
3018 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3020 DMA_REGPAIR_LE(sge->addr, vaddr); \
3021 (sge)->length = cpu_to_le32(vlength); \
3022 (sge)->l_key = cpu_to_le32(vlkey); \
3025 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3026 const struct ib_send_wr *wr)
3031 for (i = 0; i < wr->num_sge; i++) {
3032 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3034 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3035 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3036 sge->length = cpu_to_le32(wr->sg_list[i].length);
3037 data_size += wr->sg_list[i].length;
3041 *wqe_size += wr->num_sge;
3046 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3048 struct rdma_sq_rdma_wqe_1st *rwqe,
3049 struct rdma_sq_rdma_wqe_2nd *rwqe2,
3050 const struct ib_send_wr *wr,
3051 const struct ib_send_wr **bad_wr)
3053 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3054 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3056 if (wr->send_flags & IB_SEND_INLINE &&
3057 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3058 wr->opcode == IB_WR_RDMA_WRITE)) {
3061 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3062 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3063 bad_wr, &rwqe->flags, flags);
3066 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3069 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3071 struct rdma_sq_send_wqe_1st *swqe,
3072 struct rdma_sq_send_wqe_2st *swqe2,
3073 const struct ib_send_wr *wr,
3074 const struct ib_send_wr **bad_wr)
3076 memset(swqe2, 0, sizeof(*swqe2));
3077 if (wr->send_flags & IB_SEND_INLINE) {
3080 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3081 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3082 bad_wr, &swqe->flags, flags);
3085 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3088 static int qedr_prepare_reg(struct qedr_qp *qp,
3089 struct rdma_sq_fmr_wqe_1st *fwqe1,
3090 const struct ib_reg_wr *wr)
3092 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3093 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3095 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3096 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3097 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3098 fwqe1->l_key = wr->key;
3100 fwqe2->access_ctrl = 0;
3102 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3103 !!(wr->access & IB_ACCESS_REMOTE_READ));
3104 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3105 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3106 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3107 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3108 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3109 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3110 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3111 fwqe2->fmr_ctrl = 0;
3113 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3114 ilog2(mr->ibmr.page_size) - 12);
3116 fwqe2->length_hi = 0;
3117 fwqe2->length_lo = mr->ibmr.length;
3118 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3119 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3121 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3126 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3129 case IB_WR_RDMA_WRITE:
3130 case IB_WR_RDMA_WRITE_WITH_IMM:
3131 return IB_WC_RDMA_WRITE;
3132 case IB_WR_SEND_WITH_IMM:
3134 case IB_WR_SEND_WITH_INV:
3136 case IB_WR_RDMA_READ:
3137 case IB_WR_RDMA_READ_WITH_INV:
3138 return IB_WC_RDMA_READ;
3139 case IB_WR_ATOMIC_CMP_AND_SWP:
3140 return IB_WC_COMP_SWAP;
3141 case IB_WR_ATOMIC_FETCH_AND_ADD:
3142 return IB_WC_FETCH_ADD;
3144 return IB_WC_REG_MR;
3145 case IB_WR_LOCAL_INV:
3146 return IB_WC_LOCAL_INV;
3152 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3153 const struct ib_send_wr *wr)
3155 int wq_is_full, err_wr, pbl_is_full;
3156 struct qedr_dev *dev = qp->dev;
3158 /* prevent SQ overflow and/or processing of a bad WR */
3159 err_wr = wr->num_sge > qp->sq.max_sges;
3160 wq_is_full = qedr_wq_is_full(&qp->sq);
3161 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3162 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3163 if (wq_is_full || err_wr || pbl_is_full) {
3164 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3166 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3168 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3171 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3173 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3175 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3179 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3181 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3183 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3190 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3191 const struct ib_send_wr **bad_wr)
3193 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3194 struct qedr_qp *qp = get_qedr_qp(ibqp);
3195 struct rdma_sq_atomic_wqe_1st *awqe1;
3196 struct rdma_sq_atomic_wqe_2nd *awqe2;
3197 struct rdma_sq_atomic_wqe_3rd *awqe3;
3198 struct rdma_sq_send_wqe_2st *swqe2;
3199 struct rdma_sq_local_inv_wqe *iwqe;
3200 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3201 struct rdma_sq_send_wqe_1st *swqe;
3202 struct rdma_sq_rdma_wqe_1st *rwqe;
3203 struct rdma_sq_fmr_wqe_1st *fwqe1;
3204 struct rdma_sq_common_wqe *wqe;
3209 if (!qedr_can_post_send(qp, wr)) {
3214 wqe = qed_chain_produce(&qp->sq.pbl);
3215 qp->wqe_wr_id[qp->sq.prod].signaled =
3216 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3219 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3220 !!(wr->send_flags & IB_SEND_SOLICITED));
3221 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3222 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3223 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3224 !!(wr->send_flags & IB_SEND_FENCE));
3225 wqe->prev_wqe_size = qp->prev_wqe_size;
3227 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3229 switch (wr->opcode) {
3230 case IB_WR_SEND_WITH_IMM:
3231 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3236 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3237 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3239 swqe2 = qed_chain_produce(&qp->sq.pbl);
3241 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3242 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3244 swqe->length = cpu_to_le32(length);
3245 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3246 qp->prev_wqe_size = swqe->wqe_size;
3247 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3250 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3251 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3254 swqe2 = qed_chain_produce(&qp->sq.pbl);
3255 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3257 swqe->length = cpu_to_le32(length);
3258 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3259 qp->prev_wqe_size = swqe->wqe_size;
3260 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3262 case IB_WR_SEND_WITH_INV:
3263 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3264 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3265 swqe2 = qed_chain_produce(&qp->sq.pbl);
3267 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3268 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3270 swqe->length = cpu_to_le32(length);
3271 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3272 qp->prev_wqe_size = swqe->wqe_size;
3273 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3276 case IB_WR_RDMA_WRITE_WITH_IMM:
3277 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3282 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3283 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3286 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3287 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3288 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3290 rwqe->length = cpu_to_le32(length);
3291 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3292 qp->prev_wqe_size = rwqe->wqe_size;
3293 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3295 case IB_WR_RDMA_WRITE:
3296 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3297 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3300 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3301 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3303 rwqe->length = cpu_to_le32(length);
3304 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3305 qp->prev_wqe_size = rwqe->wqe_size;
3306 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3308 case IB_WR_RDMA_READ_WITH_INV:
3309 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3310 /* fallthrough -- same is identical to RDMA READ */
3312 case IB_WR_RDMA_READ:
3313 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3314 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3317 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3318 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3320 rwqe->length = cpu_to_le32(length);
3321 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3322 qp->prev_wqe_size = rwqe->wqe_size;
3323 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3326 case IB_WR_ATOMIC_CMP_AND_SWP:
3327 case IB_WR_ATOMIC_FETCH_AND_ADD:
3328 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3329 awqe1->wqe_size = 4;
3331 awqe2 = qed_chain_produce(&qp->sq.pbl);
3332 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3333 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3335 awqe3 = qed_chain_produce(&qp->sq.pbl);
3337 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3338 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3339 DMA_REGPAIR_LE(awqe3->swap_data,
3340 atomic_wr(wr)->compare_add);
3342 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3343 DMA_REGPAIR_LE(awqe3->swap_data,
3344 atomic_wr(wr)->swap);
3345 DMA_REGPAIR_LE(awqe3->cmp_data,
3346 atomic_wr(wr)->compare_add);
3349 qedr_prepare_sq_sges(qp, NULL, wr);
3351 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3352 qp->prev_wqe_size = awqe1->wqe_size;
3355 case IB_WR_LOCAL_INV:
3356 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3359 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3360 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3361 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3362 qp->prev_wqe_size = iwqe->wqe_size;
3365 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3366 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3367 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3368 fwqe1->wqe_size = 2;
3370 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3372 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3377 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3378 qp->prev_wqe_size = fwqe1->wqe_size;
3381 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3390 /* Restore prod to its position before
3391 * this WR was processed
3393 value = le16_to_cpu(qp->sq.db_data.data.value);
3394 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3396 /* Restore prev_wqe_size */
3397 qp->prev_wqe_size = wqe->prev_wqe_size;
3399 DP_ERR(dev, "POST SEND FAILED\n");
3405 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3406 const struct ib_send_wr **bad_wr)
3408 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3409 struct qedr_qp *qp = get_qedr_qp(ibqp);
3410 unsigned long flags;
3415 if (qp->qp_type == IB_QPT_GSI)
3416 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3418 spin_lock_irqsave(&qp->q_lock, flags);
3420 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3421 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3422 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3423 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3424 spin_unlock_irqrestore(&qp->q_lock, flags);
3426 DP_DEBUG(dev, QEDR_MSG_CQ,
3427 "QP in wrong state! QP icid=0x%x state %d\n",
3428 qp->icid, qp->state);
3434 rc = __qedr_post_send(ibqp, wr, bad_wr);
3438 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3440 qedr_inc_sw_prod(&qp->sq);
3442 qp->sq.db_data.data.value++;
3448 * If there was a failure in the first WR then it will be triggered in
3449 * vane. However this is not harmful (as long as the producer value is
3450 * unchanged). For performance reasons we avoid checking for this
3451 * redundant doorbell.
3453 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3454 * soon as we give the doorbell, we could get a completion
3455 * for this wr, therefore we need to make sure that the
3456 * memory is updated before giving the doorbell.
3457 * During qedr_poll_cq, rmb is called before accessing the
3458 * cqe. This covers for the smp_rmb as well.
3461 writel(qp->sq.db_data.raw, qp->sq.db);
3463 spin_unlock_irqrestore(&qp->q_lock, flags);
3468 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3472 /* Calculate number of elements used based on producer
3473 * count and consumer count and subtract it from max
3474 * work request supported so that we get elements left.
3476 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3478 return hw_srq->max_wr - used;
3481 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3482 const struct ib_recv_wr **bad_wr)
3484 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3485 struct qedr_srq_hwq_info *hw_srq;
3486 struct qedr_dev *dev = srq->dev;
3487 struct qed_chain *pbl;
3488 unsigned long flags;
3493 spin_lock_irqsave(&srq->lock, flags);
3495 hw_srq = &srq->hw_srq;
3496 pbl = &srq->hw_srq.pbl;
3498 struct rdma_srq_wqe_header *hdr;
3501 if (!qedr_srq_elem_left(hw_srq) ||
3502 wr->num_sge > srq->hw_srq.max_sges) {
3503 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3504 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3505 wr->num_sge, srq->hw_srq.max_sges);
3511 hdr = qed_chain_produce(pbl);
3512 num_sge = wr->num_sge;
3513 /* Set number of sge and work request id in header */
3514 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3516 srq->hw_srq.wr_prod_cnt++;
3520 DP_DEBUG(dev, QEDR_MSG_SRQ,
3521 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3522 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3524 for (i = 0; i < wr->num_sge; i++) {
3525 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3527 /* Set SGE length, lkey and address */
3528 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3529 wr->sg_list[i].length, wr->sg_list[i].lkey);
3531 DP_DEBUG(dev, QEDR_MSG_SRQ,
3532 "[%d]: len %d key %x addr %x:%x\n",
3533 i, srq_sge->length, srq_sge->l_key,
3534 srq_sge->addr.hi, srq_sge->addr.lo);
3538 /* Flush WQE and SGE information before
3539 * updating producer.
3543 /* SRQ producer is 8 bytes. Need to update SGE producer index
3544 * in first 4 bytes and need to update WQE producer in
3547 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3548 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3549 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3552 /* Flush producer after updating it. */
3557 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3558 qed_chain_get_elem_left(pbl));
3559 spin_unlock_irqrestore(&srq->lock, flags);
3564 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3565 const struct ib_recv_wr **bad_wr)
3567 struct qedr_qp *qp = get_qedr_qp(ibqp);
3568 struct qedr_dev *dev = qp->dev;
3569 unsigned long flags;
3572 if (qp->qp_type == IB_QPT_GSI)
3573 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3575 spin_lock_irqsave(&qp->q_lock, flags);
3577 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3578 spin_unlock_irqrestore(&qp->q_lock, flags);
3586 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3587 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3588 wr->num_sge > qp->rq.max_sges) {
3589 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3590 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3591 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3597 for (i = 0; i < wr->num_sge; i++) {
3599 struct rdma_rq_sge *rqe =
3600 qed_chain_produce(&qp->rq.pbl);
3602 /* First one must include the number
3603 * of SGE in the list
3606 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3609 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3610 wr->sg_list[i].lkey);
3612 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3613 wr->sg_list[i].length, flags);
3616 /* Special case of no sges. FW requires between 1-4 sges...
3617 * in this case we need to post 1 sge with length zero. this is
3618 * because rdma write with immediate consumes an RQ.
3622 struct rdma_rq_sge *rqe =
3623 qed_chain_produce(&qp->rq.pbl);
3625 /* First one must include the number
3626 * of SGE in the list
3628 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3629 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3631 RQ_SGE_SET(rqe, 0, 0, flags);
3635 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3636 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3638 qedr_inc_sw_prod(&qp->rq);
3640 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3641 * soon as we give the doorbell, we could get a completion
3642 * for this wr, therefore we need to make sure that the
3643 * memory is update before giving the doorbell.
3644 * During qedr_poll_cq, rmb is called before accessing the
3645 * cqe. This covers for the smp_rmb as well.
3649 qp->rq.db_data.data.value++;
3651 writel(qp->rq.db_data.raw, qp->rq.db);
3653 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3654 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3660 spin_unlock_irqrestore(&qp->q_lock, flags);
3665 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3667 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3669 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3673 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3675 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3678 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3679 resp_cqe->qp_handle.lo,
3684 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3686 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3688 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3691 /* Return latest CQE (needs processing) */
3692 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3694 return cq->latest_cqe;
3697 /* In fmr we need to increase the number of fmr completed counter for the fmr
3698 * algorithm determining whether we can free a pbl or not.
3699 * we need to perform this whether the work request was signaled or not. for
3700 * this purpose we call this function from the condition that checks if a wr
3701 * should be skipped, to make sure we don't miss it ( possibly this fmr
3702 * operation was not signalted)
3704 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3706 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3707 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3710 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3711 struct qedr_cq *cq, int num_entries,
3712 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3717 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3718 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3719 qedr_chk_if_fmr(qp);
3725 wc->status = status;
3728 wc->src_qp = qp->id;
3731 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3732 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3734 switch (wc->opcode) {
3735 case IB_WC_RDMA_WRITE:
3736 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3738 case IB_WC_COMP_SWAP:
3739 case IB_WC_FETCH_ADD:
3743 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3745 case IB_WC_RDMA_READ:
3747 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3757 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3758 qed_chain_consume(&qp->sq.pbl);
3759 qedr_inc_sw_cons(&qp->sq);
3765 static int qedr_poll_cq_req(struct qedr_dev *dev,
3766 struct qedr_qp *qp, struct qedr_cq *cq,
3767 int num_entries, struct ib_wc *wc,
3768 struct rdma_cqe_requester *req)
3772 switch (req->status) {
3773 case RDMA_CQE_REQ_STS_OK:
3774 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3777 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3778 if (qp->state != QED_ROCE_QP_STATE_ERR)
3779 DP_DEBUG(dev, QEDR_MSG_CQ,
3780 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3781 cq->icid, qp->icid);
3782 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3783 IB_WC_WR_FLUSH_ERR, 1);
3786 /* process all WQE before the cosumer */
3787 qp->state = QED_ROCE_QP_STATE_ERR;
3788 cnt = process_req(dev, qp, cq, num_entries, wc,
3789 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3791 /* if we have extra WC fill it with actual error info */
3792 if (cnt < num_entries) {
3793 enum ib_wc_status wc_status;
3795 switch (req->status) {
3796 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3798 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3799 cq->icid, qp->icid);
3800 wc_status = IB_WC_BAD_RESP_ERR;
3802 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3804 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3805 cq->icid, qp->icid);
3806 wc_status = IB_WC_LOC_LEN_ERR;
3808 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3810 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3811 cq->icid, qp->icid);
3812 wc_status = IB_WC_LOC_QP_OP_ERR;
3814 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3816 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3817 cq->icid, qp->icid);
3818 wc_status = IB_WC_LOC_PROT_ERR;
3820 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3822 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3823 cq->icid, qp->icid);
3824 wc_status = IB_WC_MW_BIND_ERR;
3826 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3828 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3829 cq->icid, qp->icid);
3830 wc_status = IB_WC_REM_INV_REQ_ERR;
3832 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3834 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3835 cq->icid, qp->icid);
3836 wc_status = IB_WC_REM_ACCESS_ERR;
3838 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3840 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3841 cq->icid, qp->icid);
3842 wc_status = IB_WC_REM_OP_ERR;
3844 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3846 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3847 cq->icid, qp->icid);
3848 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3850 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3852 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3853 cq->icid, qp->icid);
3854 wc_status = IB_WC_RETRY_EXC_ERR;
3858 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3859 cq->icid, qp->icid);
3860 wc_status = IB_WC_GENERAL_ERR;
3862 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3870 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3873 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3874 return IB_WC_LOC_ACCESS_ERR;
3875 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3876 return IB_WC_LOC_LEN_ERR;
3877 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3878 return IB_WC_LOC_QP_OP_ERR;
3879 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3880 return IB_WC_LOC_PROT_ERR;
3881 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3882 return IB_WC_MW_BIND_ERR;
3883 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3884 return IB_WC_REM_INV_RD_REQ_ERR;
3885 case RDMA_CQE_RESP_STS_OK:
3886 return IB_WC_SUCCESS;
3888 return IB_WC_GENERAL_ERR;
3892 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3895 wc->status = IB_WC_SUCCESS;
3896 wc->byte_len = le32_to_cpu(resp->length);
3898 if (resp->flags & QEDR_RESP_IMM) {
3899 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
3900 wc->wc_flags |= IB_WC_WITH_IMM;
3902 if (resp->flags & QEDR_RESP_RDMA)
3903 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3905 if (resp->flags & QEDR_RESP_INV)
3908 } else if (resp->flags & QEDR_RESP_INV) {
3909 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3910 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3912 if (resp->flags & QEDR_RESP_RDMA)
3915 } else if (resp->flags & QEDR_RESP_RDMA) {
3922 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3923 struct qedr_cq *cq, struct ib_wc *wc,
3924 struct rdma_cqe_responder *resp, u64 wr_id)
3926 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3927 wc->opcode = IB_WC_RECV;
3930 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3931 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3933 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3934 cq, cq->icid, resp->flags);
3937 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3938 if (wc->status == IB_WC_GENERAL_ERR)
3940 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3941 cq, cq->icid, resp->status);
3944 /* Fill the rest of the WC */
3946 wc->src_qp = qp->id;
3951 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3952 struct qedr_cq *cq, struct ib_wc *wc,
3953 struct rdma_cqe_responder *resp)
3955 struct qedr_srq *srq = qp->srq;
3958 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
3959 le32_to_cpu(resp->srq_wr_id.lo), u64);
3961 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3962 wc->status = IB_WC_WR_FLUSH_ERR;
3966 wc->src_qp = qp->id;
3970 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3972 srq->hw_srq.wr_cons_cnt++;
3976 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3977 struct qedr_cq *cq, struct ib_wc *wc,
3978 struct rdma_cqe_responder *resp)
3980 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3982 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3984 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3985 qed_chain_consume(&qp->rq.pbl);
3986 qedr_inc_sw_cons(&qp->rq);
3991 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3992 int num_entries, struct ib_wc *wc, u16 hw_cons)
3996 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3998 wc->status = IB_WC_WR_FLUSH_ERR;
4001 wc->src_qp = qp->id;
4003 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4008 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4009 qed_chain_consume(&qp->rq.pbl);
4010 qedr_inc_sw_cons(&qp->rq);
4016 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4017 struct rdma_cqe_responder *resp, int *update)
4019 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4025 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4026 struct qedr_cq *cq, int num_entries,
4028 struct rdma_cqe_responder *resp)
4032 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4038 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4039 struct qedr_cq *cq, int num_entries,
4040 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4045 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4046 cnt = process_resp_flush(qp, cq, num_entries, wc,
4047 resp->rq_cons_or_srq_id);
4048 try_consume_resp_cqe(cq, qp, resp, update);
4050 cnt = process_resp_one(dev, qp, cq, wc, resp);
4058 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4059 struct rdma_cqe_requester *req, int *update)
4061 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4067 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4069 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4070 struct qedr_cq *cq = get_qedr_cq(ibcq);
4071 union rdma_cqe *cqe;
4072 u32 old_cons, new_cons;
4073 unsigned long flags;
4077 if (cq->destroyed) {
4079 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4084 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4085 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4087 spin_lock_irqsave(&cq->cq_lock, flags);
4088 cqe = cq->latest_cqe;
4089 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4090 while (num_entries && is_valid_cqe(cq, cqe)) {
4094 /* prevent speculative reads of any field of CQE */
4097 qp = cqe_get_qp(cqe);
4099 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4105 switch (cqe_get_type(cqe)) {
4106 case RDMA_CQE_TYPE_REQUESTER:
4107 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4109 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4111 case RDMA_CQE_TYPE_RESPONDER_RQ:
4112 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4113 &cqe->resp, &update);
4115 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4116 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4120 case RDMA_CQE_TYPE_INVALID:
4122 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4131 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4133 cq->cq_cons += new_cons - old_cons;
4136 /* doorbell notifies abount latest VALID entry,
4137 * but chain already point to the next INVALID one
4139 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4141 spin_unlock_irqrestore(&cq->cq_lock, flags);
4145 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4147 const struct ib_wc *in_wc,
4148 const struct ib_grh *in_grh,
4149 const struct ib_mad_hdr *mad_hdr,
4150 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4151 size_t *out_mad_size, u16 *out_mad_pkey_index)
4153 struct qedr_dev *dev = get_qedr_dev(ibdev);
4155 DP_DEBUG(dev, QEDR_MSG_GSI,
4156 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4157 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4158 mad_hdr->class_specific, mad_hdr->class_version,
4159 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4160 return IB_MAD_RESULT_SUCCESS;