634d1586a1fabe4856344749d5b883c298026e81
[linux-2.6-microblaze.git] / drivers / infiniband / hw / bnxt_re / ib_verbs.c
1 /*
2  * Broadcom NetXtreme-E RoCE driver.
3  *
4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * BSD license below:
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in
21  *    the documentation and/or other materials provided with the
22  *    distribution.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  * Description: IB Verbs interpreter
37  */
38
39 #include <linux/interrupt.h>
40 #include <linux/types.h>
41 #include <linux/pci.h>
42 #include <linux/netdevice.h>
43 #include <linux/if_ether.h>
44
45 #include <rdma/ib_verbs.h>
46 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
48 #include <rdma/ib_addr.h>
49 #include <rdma/ib_mad.h>
50 #include <rdma/ib_cache.h>
51 #include <rdma/uverbs_ioctl.h>
52
53 #include "bnxt_ulp.h"
54
55 #include "roce_hsi.h"
56 #include "qplib_res.h"
57 #include "qplib_sp.h"
58 #include "qplib_fp.h"
59 #include "qplib_rcfw.h"
60
61 #include "bnxt_re.h"
62 #include "ib_verbs.h"
63 #include <rdma/bnxt_re-abi.h>
64
65 static int __from_ib_access_flags(int iflags)
66 {
67         int qflags = 0;
68
69         if (iflags & IB_ACCESS_LOCAL_WRITE)
70                 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
71         if (iflags & IB_ACCESS_REMOTE_READ)
72                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
73         if (iflags & IB_ACCESS_REMOTE_WRITE)
74                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
75         if (iflags & IB_ACCESS_REMOTE_ATOMIC)
76                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
77         if (iflags & IB_ACCESS_MW_BIND)
78                 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
79         if (iflags & IB_ZERO_BASED)
80                 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
81         if (iflags & IB_ACCESS_ON_DEMAND)
82                 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
83         return qflags;
84 };
85
86 static enum ib_access_flags __to_ib_access_flags(int qflags)
87 {
88         enum ib_access_flags iflags = 0;
89
90         if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
91                 iflags |= IB_ACCESS_LOCAL_WRITE;
92         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
93                 iflags |= IB_ACCESS_REMOTE_WRITE;
94         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
95                 iflags |= IB_ACCESS_REMOTE_READ;
96         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
97                 iflags |= IB_ACCESS_REMOTE_ATOMIC;
98         if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
99                 iflags |= IB_ACCESS_MW_BIND;
100         if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
101                 iflags |= IB_ZERO_BASED;
102         if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
103                 iflags |= IB_ACCESS_ON_DEMAND;
104         return iflags;
105 };
106
107 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
108                              struct bnxt_qplib_sge *sg_list, int num)
109 {
110         int i, total = 0;
111
112         for (i = 0; i < num; i++) {
113                 sg_list[i].addr = ib_sg_list[i].addr;
114                 sg_list[i].lkey = ib_sg_list[i].lkey;
115                 sg_list[i].size = ib_sg_list[i].length;
116                 total += sg_list[i].size;
117         }
118         return total;
119 }
120
121 /* Device */
122 int bnxt_re_query_device(struct ib_device *ibdev,
123                          struct ib_device_attr *ib_attr,
124                          struct ib_udata *udata)
125 {
126         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
127         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
128
129         memset(ib_attr, 0, sizeof(*ib_attr));
130         memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
131                min(sizeof(dev_attr->fw_ver),
132                    sizeof(ib_attr->fw_ver)));
133         bnxt_qplib_get_guid(rdev->netdev->dev_addr,
134                             (u8 *)&ib_attr->sys_image_guid);
135         ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
136         ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M;
137
138         ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
139         ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
140         ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
141         ib_attr->max_qp = dev_attr->max_qp;
142         ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
143         ib_attr->device_cap_flags =
144                                     IB_DEVICE_CURR_QP_STATE_MOD
145                                     | IB_DEVICE_RC_RNR_NAK_GEN
146                                     | IB_DEVICE_SHUTDOWN_PORT
147                                     | IB_DEVICE_SYS_IMAGE_GUID
148                                     | IB_DEVICE_LOCAL_DMA_LKEY
149                                     | IB_DEVICE_RESIZE_MAX_WR
150                                     | IB_DEVICE_PORT_ACTIVE_EVENT
151                                     | IB_DEVICE_N_NOTIFY_CQ
152                                     | IB_DEVICE_MEM_WINDOW
153                                     | IB_DEVICE_MEM_WINDOW_TYPE_2B
154                                     | IB_DEVICE_MEM_MGT_EXTENSIONS;
155         ib_attr->max_send_sge = dev_attr->max_qp_sges;
156         ib_attr->max_recv_sge = dev_attr->max_qp_sges;
157         ib_attr->max_sge_rd = dev_attr->max_qp_sges;
158         ib_attr->max_cq = dev_attr->max_cq;
159         ib_attr->max_cqe = dev_attr->max_cq_wqes;
160         ib_attr->max_mr = dev_attr->max_mr;
161         ib_attr->max_pd = dev_attr->max_pd;
162         ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
163         ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
164         ib_attr->atomic_cap = IB_ATOMIC_NONE;
165         ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
166         if (dev_attr->is_atomic) {
167                 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
168                 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
169         }
170
171         ib_attr->max_ee_rd_atom = 0;
172         ib_attr->max_res_rd_atom = 0;
173         ib_attr->max_ee_init_rd_atom = 0;
174         ib_attr->max_ee = 0;
175         ib_attr->max_rdd = 0;
176         ib_attr->max_mw = dev_attr->max_mw;
177         ib_attr->max_raw_ipv6_qp = 0;
178         ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
179         ib_attr->max_mcast_grp = 0;
180         ib_attr->max_mcast_qp_attach = 0;
181         ib_attr->max_total_mcast_qp_attach = 0;
182         ib_attr->max_ah = dev_attr->max_ah;
183
184         ib_attr->max_srq = dev_attr->max_srq;
185         ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
186         ib_attr->max_srq_sge = dev_attr->max_srq_sges;
187
188         ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
189
190         ib_attr->max_pkeys = 1;
191         ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
192         return 0;
193 }
194
195 /* Port */
196 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
197                        struct ib_port_attr *port_attr)
198 {
199         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
200         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
201
202         memset(port_attr, 0, sizeof(*port_attr));
203
204         if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
205                 port_attr->state = IB_PORT_ACTIVE;
206                 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
207         } else {
208                 port_attr->state = IB_PORT_DOWN;
209                 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
210         }
211         port_attr->max_mtu = IB_MTU_4096;
212         port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
213         port_attr->gid_tbl_len = dev_attr->max_sgid;
214         port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
215                                     IB_PORT_DEVICE_MGMT_SUP |
216                                     IB_PORT_VENDOR_CLASS_SUP;
217         port_attr->ip_gids = true;
218
219         port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
220         port_attr->bad_pkey_cntr = 0;
221         port_attr->qkey_viol_cntr = 0;
222         port_attr->pkey_tbl_len = dev_attr->max_pkey;
223         port_attr->lid = 0;
224         port_attr->sm_lid = 0;
225         port_attr->lmc = 0;
226         port_attr->max_vl_num = 4;
227         port_attr->sm_sl = 0;
228         port_attr->subnet_timeout = 0;
229         port_attr->init_type_reply = 0;
230         port_attr->active_speed = rdev->active_speed;
231         port_attr->active_width = rdev->active_width;
232
233         return 0;
234 }
235
236 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
237                                struct ib_port_immutable *immutable)
238 {
239         struct ib_port_attr port_attr;
240
241         if (bnxt_re_query_port(ibdev, port_num, &port_attr))
242                 return -EINVAL;
243
244         immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
245         immutable->gid_tbl_len = port_attr.gid_tbl_len;
246         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
247         immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
248         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
249         return 0;
250 }
251
252 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
253 {
254         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
255
256         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
257                  rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1],
258                  rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]);
259 }
260
261 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
262                        u16 index, u16 *pkey)
263 {
264         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
265
266         /* Ignore port_num */
267
268         memset(pkey, 0, sizeof(*pkey));
269         return bnxt_qplib_get_pkey(&rdev->qplib_res,
270                                    &rdev->qplib_res.pkey_tbl, index, pkey);
271 }
272
273 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
274                       int index, union ib_gid *gid)
275 {
276         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
277         int rc = 0;
278
279         /* Ignore port_num */
280         memset(gid, 0, sizeof(*gid));
281         rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
282                                  &rdev->qplib_res.sgid_tbl, index,
283                                  (struct bnxt_qplib_gid *)gid);
284         return rc;
285 }
286
287 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
288 {
289         int rc = 0;
290         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
291         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
292         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
293         struct bnxt_qplib_gid *gid_to_del;
294         u16 vlan_id = 0xFFFF;
295
296         /* Delete the entry from the hardware */
297         ctx = *context;
298         if (!ctx)
299                 return -EINVAL;
300
301         if (sgid_tbl && sgid_tbl->active) {
302                 if (ctx->idx >= sgid_tbl->max)
303                         return -EINVAL;
304                 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
305                 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
306                 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
307                  * or via the ib_unregister_device path. In the former case QP1
308                  * may not be destroyed yet, in which case just return as FW
309                  * needs that entry to be present and will fail it's deletion.
310                  * We could get invoked again after QP1 is destroyed OR get an
311                  * ADD_GID call with a different GID value for the same index
312                  * where we issue MODIFY_GID cmd to update the GID entry -- TBD
313                  */
314                 if (ctx->idx == 0 &&
315                     rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
316                     ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
317                         ibdev_dbg(&rdev->ibdev,
318                                   "Trying to delete GID0 while QP1 is alive\n");
319                         return -EFAULT;
320                 }
321                 ctx->refcnt--;
322                 if (!ctx->refcnt) {
323                         rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
324                                                  vlan_id,  true);
325                         if (rc) {
326                                 ibdev_err(&rdev->ibdev,
327                                           "Failed to remove GID: %#x", rc);
328                         } else {
329                                 ctx_tbl = sgid_tbl->ctx;
330                                 ctx_tbl[ctx->idx] = NULL;
331                                 kfree(ctx);
332                         }
333                 }
334         } else {
335                 return -EINVAL;
336         }
337         return rc;
338 }
339
340 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
341 {
342         int rc;
343         u32 tbl_idx = 0;
344         u16 vlan_id = 0xFFFF;
345         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
346         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
347         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
348
349         rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
350         if (rc)
351                 return rc;
352
353         rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
354                                  rdev->qplib_res.netdev->dev_addr,
355                                  vlan_id, true, &tbl_idx);
356         if (rc == -EALREADY) {
357                 ctx_tbl = sgid_tbl->ctx;
358                 ctx_tbl[tbl_idx]->refcnt++;
359                 *context = ctx_tbl[tbl_idx];
360                 return 0;
361         }
362
363         if (rc < 0) {
364                 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
365                 return rc;
366         }
367
368         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
369         if (!ctx)
370                 return -ENOMEM;
371         ctx_tbl = sgid_tbl->ctx;
372         ctx->idx = tbl_idx;
373         ctx->refcnt = 1;
374         ctx_tbl[tbl_idx] = ctx;
375         *context = ctx;
376
377         return rc;
378 }
379
380 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
381                                             u32 port_num)
382 {
383         return IB_LINK_LAYER_ETHERNET;
384 }
385
386 #define BNXT_RE_FENCE_PBL_SIZE  DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
387
388 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
389 {
390         struct bnxt_re_fence_data *fence = &pd->fence;
391         struct ib_mr *ib_mr = &fence->mr->ib_mr;
392         struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
393
394         memset(wqe, 0, sizeof(*wqe));
395         wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
396         wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
397         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
398         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
399         wqe->bind.zero_based = false;
400         wqe->bind.parent_l_key = ib_mr->lkey;
401         wqe->bind.va = (u64)(unsigned long)fence->va;
402         wqe->bind.length = fence->size;
403         wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
404         wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
405
406         /* Save the initial rkey in fence structure for now;
407          * wqe->bind.r_key will be set at (re)bind time.
408          */
409         fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
410 }
411
412 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
413 {
414         struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
415                                              qplib_qp);
416         struct ib_pd *ib_pd = qp->ib_qp.pd;
417         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
418         struct bnxt_re_fence_data *fence = &pd->fence;
419         struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
420         struct bnxt_qplib_swqe wqe;
421         int rc;
422
423         memcpy(&wqe, fence_wqe, sizeof(wqe));
424         wqe.bind.r_key = fence->bind_rkey;
425         fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
426
427         ibdev_dbg(&qp->rdev->ibdev,
428                   "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
429                 wqe.bind.r_key, qp->qplib_qp.id, pd);
430         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
431         if (rc) {
432                 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
433                 return rc;
434         }
435         bnxt_qplib_post_send_db(&qp->qplib_qp);
436
437         return rc;
438 }
439
440 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
441 {
442         struct bnxt_re_fence_data *fence = &pd->fence;
443         struct bnxt_re_dev *rdev = pd->rdev;
444         struct device *dev = &rdev->en_dev->pdev->dev;
445         struct bnxt_re_mr *mr = fence->mr;
446
447         if (fence->mw) {
448                 bnxt_re_dealloc_mw(fence->mw);
449                 fence->mw = NULL;
450         }
451         if (mr) {
452                 if (mr->ib_mr.rkey)
453                         bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
454                                              true);
455                 if (mr->ib_mr.lkey)
456                         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
457                 kfree(mr);
458                 fence->mr = NULL;
459         }
460         if (fence->dma_addr) {
461                 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
462                                  DMA_BIDIRECTIONAL);
463                 fence->dma_addr = 0;
464         }
465 }
466
467 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
468 {
469         int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
470         struct bnxt_re_fence_data *fence = &pd->fence;
471         struct bnxt_re_dev *rdev = pd->rdev;
472         struct device *dev = &rdev->en_dev->pdev->dev;
473         struct bnxt_re_mr *mr = NULL;
474         dma_addr_t dma_addr = 0;
475         struct ib_mw *mw;
476         int rc;
477
478         dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
479                                   DMA_BIDIRECTIONAL);
480         rc = dma_mapping_error(dev, dma_addr);
481         if (rc) {
482                 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
483                 rc = -EIO;
484                 fence->dma_addr = 0;
485                 goto fail;
486         }
487         fence->dma_addr = dma_addr;
488
489         /* Allocate a MR */
490         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
491         if (!mr) {
492                 rc = -ENOMEM;
493                 goto fail;
494         }
495         fence->mr = mr;
496         mr->rdev = rdev;
497         mr->qplib_mr.pd = &pd->qplib_pd;
498         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
499         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
500         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
501         if (rc) {
502                 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
503                 goto fail;
504         }
505
506         /* Register MR */
507         mr->ib_mr.lkey = mr->qplib_mr.lkey;
508         mr->qplib_mr.va = (u64)(unsigned long)fence->va;
509         mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
510         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
511                                BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
512         if (rc) {
513                 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
514                 goto fail;
515         }
516         mr->ib_mr.rkey = mr->qplib_mr.rkey;
517
518         /* Create a fence MW only for kernel consumers */
519         mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
520         if (IS_ERR(mw)) {
521                 ibdev_err(&rdev->ibdev,
522                           "Failed to create fence-MW for PD: %p\n", pd);
523                 rc = PTR_ERR(mw);
524                 goto fail;
525         }
526         fence->mw = mw;
527
528         bnxt_re_create_fence_wqe(pd);
529         return 0;
530
531 fail:
532         bnxt_re_destroy_fence_mr(pd);
533         return rc;
534 }
535
536 /* Protection Domains */
537 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
538 {
539         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
540         struct bnxt_re_dev *rdev = pd->rdev;
541
542         bnxt_re_destroy_fence_mr(pd);
543
544         if (pd->qplib_pd.id)
545                 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
546                                       &pd->qplib_pd);
547         return 0;
548 }
549
550 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
551 {
552         struct ib_device *ibdev = ibpd->device;
553         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
554         struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
555                 udata, struct bnxt_re_ucontext, ib_uctx);
556         struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
557         int rc;
558
559         pd->rdev = rdev;
560         if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
561                 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
562                 rc = -ENOMEM;
563                 goto fail;
564         }
565
566         if (udata) {
567                 struct bnxt_re_pd_resp resp;
568
569                 if (!ucntx->dpi.dbr) {
570                         /* Allocate DPI in alloc_pd to avoid failing of
571                          * ibv_devinfo and family of application when DPIs
572                          * are depleted.
573                          */
574                         if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
575                                                  &ucntx->dpi, ucntx)) {
576                                 rc = -ENOMEM;
577                                 goto dbfail;
578                         }
579                 }
580
581                 resp.pdid = pd->qplib_pd.id;
582                 /* Still allow mapping this DBR to the new user PD. */
583                 resp.dpi = ucntx->dpi.dpi;
584                 resp.dbr = (u64)ucntx->dpi.umdbr;
585
586                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
587                 if (rc) {
588                         ibdev_err(&rdev->ibdev,
589                                   "Failed to copy user response\n");
590                         goto dbfail;
591                 }
592         }
593
594         if (!udata)
595                 if (bnxt_re_create_fence_mr(pd))
596                         ibdev_warn(&rdev->ibdev,
597                                    "Failed to create Fence-MR\n");
598         return 0;
599 dbfail:
600         bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
601                               &pd->qplib_pd);
602 fail:
603         return rc;
604 }
605
606 /* Address Handles */
607 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
608 {
609         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
610         struct bnxt_re_dev *rdev = ah->rdev;
611
612         bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
613                               !(flags & RDMA_DESTROY_AH_SLEEPABLE));
614         return 0;
615 }
616
617 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
618 {
619         u8 nw_type;
620
621         switch (ntype) {
622         case RDMA_NETWORK_IPV4:
623                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
624                 break;
625         case RDMA_NETWORK_IPV6:
626                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
627                 break;
628         default:
629                 nw_type = CMDQ_CREATE_AH_TYPE_V1;
630                 break;
631         }
632         return nw_type;
633 }
634
635 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
636                       struct ib_udata *udata)
637 {
638         struct ib_pd *ib_pd = ib_ah->pd;
639         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
640         struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
641         const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
642         struct bnxt_re_dev *rdev = pd->rdev;
643         const struct ib_gid_attr *sgid_attr;
644         struct bnxt_re_gid_ctx *ctx;
645         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
646         u8 nw_type;
647         int rc;
648
649         if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
650                 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
651                 return -EINVAL;
652         }
653
654         ah->rdev = rdev;
655         ah->qplib_ah.pd = &pd->qplib_pd;
656
657         /* Supply the configuration for the HW */
658         memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
659                sizeof(union ib_gid));
660         sgid_attr = grh->sgid_attr;
661         /* Get the HW context of the GID. The reference
662          * of GID table entry is already taken by the caller.
663          */
664         ctx = rdma_read_gid_hw_context(sgid_attr);
665         ah->qplib_ah.sgid_index = ctx->idx;
666         ah->qplib_ah.host_sgid_index = grh->sgid_index;
667         ah->qplib_ah.traffic_class = grh->traffic_class;
668         ah->qplib_ah.flow_label = grh->flow_label;
669         ah->qplib_ah.hop_limit = grh->hop_limit;
670         ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
671
672         /* Get network header type for this GID */
673         nw_type = rdma_gid_attr_network_type(sgid_attr);
674         ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
675
676         memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
677         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
678                                   !(init_attr->flags &
679                                     RDMA_CREATE_AH_SLEEPABLE));
680         if (rc) {
681                 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
682                 return rc;
683         }
684
685         /* Write AVID to shared page. */
686         if (udata) {
687                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
688                         udata, struct bnxt_re_ucontext, ib_uctx);
689                 unsigned long flag;
690                 u32 *wrptr;
691
692                 spin_lock_irqsave(&uctx->sh_lock, flag);
693                 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
694                 *wrptr = ah->qplib_ah.id;
695                 wmb(); /* make sure cache is updated. */
696                 spin_unlock_irqrestore(&uctx->sh_lock, flag);
697         }
698
699         return 0;
700 }
701
702 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
703 {
704         return 0;
705 }
706
707 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
708 {
709         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
710
711         ah_attr->type = ib_ah->type;
712         rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
713         memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
714         rdma_ah_set_grh(ah_attr, NULL, 0,
715                         ah->qplib_ah.host_sgid_index,
716                         0, ah->qplib_ah.traffic_class);
717         rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
718         rdma_ah_set_port_num(ah_attr, 1);
719         rdma_ah_set_static_rate(ah_attr, 0);
720         return 0;
721 }
722
723 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
724         __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
725 {
726         unsigned long flags;
727
728         spin_lock_irqsave(&qp->scq->cq_lock, flags);
729         if (qp->rcq != qp->scq)
730                 spin_lock(&qp->rcq->cq_lock);
731         else
732                 __acquire(&qp->rcq->cq_lock);
733
734         return flags;
735 }
736
737 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
738                         unsigned long flags)
739         __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
740 {
741         if (qp->rcq != qp->scq)
742                 spin_unlock(&qp->rcq->cq_lock);
743         else
744                 __release(&qp->rcq->cq_lock);
745         spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
746 }
747
748 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
749 {
750         struct bnxt_re_qp *gsi_sqp;
751         struct bnxt_re_ah *gsi_sah;
752         struct bnxt_re_dev *rdev;
753         int rc = 0;
754
755         rdev = qp->rdev;
756         gsi_sqp = rdev->gsi_ctx.gsi_sqp;
757         gsi_sah = rdev->gsi_ctx.gsi_sah;
758
759         ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
760         bnxt_qplib_destroy_ah(&rdev->qplib_res,
761                               &gsi_sah->qplib_ah,
762                               true);
763         bnxt_qplib_clean_qp(&qp->qplib_qp);
764
765         ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
766         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
767         if (rc) {
768                 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
769                 goto fail;
770         }
771         bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
772
773         /* remove from active qp list */
774         mutex_lock(&rdev->qp_lock);
775         list_del(&gsi_sqp->list);
776         mutex_unlock(&rdev->qp_lock);
777         atomic_dec(&rdev->qp_count);
778
779         kfree(rdev->gsi_ctx.sqp_tbl);
780         kfree(gsi_sah);
781         kfree(gsi_sqp);
782         rdev->gsi_ctx.gsi_sqp = NULL;
783         rdev->gsi_ctx.gsi_sah = NULL;
784         rdev->gsi_ctx.sqp_tbl = NULL;
785
786         return 0;
787 fail:
788         return rc;
789 }
790
791 /* Queue Pairs */
792 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
793 {
794         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
795         struct bnxt_re_dev *rdev = qp->rdev;
796         unsigned int flags;
797         int rc;
798
799         bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
800
801         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
802         if (rc) {
803                 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
804                 return rc;
805         }
806
807         if (rdma_is_kernel_res(&qp->ib_qp.res)) {
808                 flags = bnxt_re_lock_cqs(qp);
809                 bnxt_qplib_clean_qp(&qp->qplib_qp);
810                 bnxt_re_unlock_cqs(qp, flags);
811         }
812
813         bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
814
815         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
816                 rc = bnxt_re_destroy_gsi_sqp(qp);
817                 if (rc)
818                         return rc;
819         }
820
821         mutex_lock(&rdev->qp_lock);
822         list_del(&qp->list);
823         mutex_unlock(&rdev->qp_lock);
824         atomic_dec(&rdev->qp_count);
825
826         ib_umem_release(qp->rumem);
827         ib_umem_release(qp->sumem);
828
829         return 0;
830 }
831
832 static u8 __from_ib_qp_type(enum ib_qp_type type)
833 {
834         switch (type) {
835         case IB_QPT_GSI:
836                 return CMDQ_CREATE_QP1_TYPE_GSI;
837         case IB_QPT_RC:
838                 return CMDQ_CREATE_QP_TYPE_RC;
839         case IB_QPT_UD:
840                 return CMDQ_CREATE_QP_TYPE_UD;
841         default:
842                 return IB_QPT_MAX;
843         }
844 }
845
846 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
847                                    int rsge, int max)
848 {
849         if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
850                 rsge = max;
851         return bnxt_re_get_rwqe_size(rsge);
852 }
853
854 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
855 {
856         u16 wqe_size, calc_ils;
857
858         wqe_size = bnxt_re_get_swqe_size(nsge);
859         if (ilsize) {
860                 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
861                 wqe_size = max_t(u16, calc_ils, wqe_size);
862                 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
863         }
864         return wqe_size;
865 }
866
867 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
868                                    struct ib_qp_init_attr *init_attr)
869 {
870         struct bnxt_qplib_dev_attr *dev_attr;
871         struct bnxt_qplib_qp *qplqp;
872         struct bnxt_re_dev *rdev;
873         struct bnxt_qplib_q *sq;
874         int align, ilsize;
875
876         rdev = qp->rdev;
877         qplqp = &qp->qplib_qp;
878         sq = &qplqp->sq;
879         dev_attr = &rdev->dev_attr;
880
881         align = sizeof(struct sq_send_hdr);
882         ilsize = ALIGN(init_attr->cap.max_inline_data, align);
883
884         sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
885         if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
886                 return -EINVAL;
887         /* For gen p4 and gen p5 backward compatibility mode
888          * wqe size is fixed to 128 bytes
889          */
890         if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
891                         qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
892                 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
893
894         if (init_attr->cap.max_inline_data) {
895                 qplqp->max_inline_data = sq->wqe_size -
896                         sizeof(struct sq_send_hdr);
897                 init_attr->cap.max_inline_data = qplqp->max_inline_data;
898                 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
899                         sq->max_sge = qplqp->max_inline_data /
900                                 sizeof(struct sq_sge);
901         }
902
903         return 0;
904 }
905
906 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
907                                 struct bnxt_re_qp *qp, struct ib_udata *udata)
908 {
909         struct bnxt_qplib_qp *qplib_qp;
910         struct bnxt_re_ucontext *cntx;
911         struct bnxt_re_qp_req ureq;
912         int bytes = 0, psn_sz;
913         struct ib_umem *umem;
914         int psn_nume;
915
916         qplib_qp = &qp->qplib_qp;
917         cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
918                                          ib_uctx);
919         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
920                 return -EFAULT;
921
922         bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
923         /* Consider mapping PSN search memory only for RC QPs. */
924         if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
925                 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
926                                                    sizeof(struct sq_psn_search_ext) :
927                                                    sizeof(struct sq_psn_search);
928                 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
929                             qplib_qp->sq.max_wqe :
930                             ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
931                               sizeof(struct bnxt_qplib_sge));
932                 bytes += (psn_nume * psn_sz);
933         }
934
935         bytes = PAGE_ALIGN(bytes);
936         umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
937                            IB_ACCESS_LOCAL_WRITE);
938         if (IS_ERR(umem))
939                 return PTR_ERR(umem);
940
941         qp->sumem = umem;
942         qplib_qp->sq.sg_info.umem = umem;
943         qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
944         qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
945         qplib_qp->qp_handle = ureq.qp_handle;
946
947         if (!qp->qplib_qp.srq) {
948                 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
949                 bytes = PAGE_ALIGN(bytes);
950                 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes,
951                                    IB_ACCESS_LOCAL_WRITE);
952                 if (IS_ERR(umem))
953                         goto rqfail;
954                 qp->rumem = umem;
955                 qplib_qp->rq.sg_info.umem = umem;
956                 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
957                 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
958         }
959
960         qplib_qp->dpi = &cntx->dpi;
961         return 0;
962 rqfail:
963         ib_umem_release(qp->sumem);
964         qp->sumem = NULL;
965         memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
966
967         return PTR_ERR(umem);
968 }
969
970 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
971                                 (struct bnxt_re_pd *pd,
972                                  struct bnxt_qplib_res *qp1_res,
973                                  struct bnxt_qplib_qp *qp1_qp)
974 {
975         struct bnxt_re_dev *rdev = pd->rdev;
976         struct bnxt_re_ah *ah;
977         union ib_gid sgid;
978         int rc;
979
980         ah = kzalloc(sizeof(*ah), GFP_KERNEL);
981         if (!ah)
982                 return NULL;
983
984         ah->rdev = rdev;
985         ah->qplib_ah.pd = &pd->qplib_pd;
986
987         rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
988         if (rc)
989                 goto fail;
990
991         /* supply the dgid data same as sgid */
992         memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
993                sizeof(union ib_gid));
994         ah->qplib_ah.sgid_index = 0;
995
996         ah->qplib_ah.traffic_class = 0;
997         ah->qplib_ah.flow_label = 0;
998         ah->qplib_ah.hop_limit = 1;
999         ah->qplib_ah.sl = 0;
1000         /* Have DMAC same as SMAC */
1001         ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1002
1003         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1004         if (rc) {
1005                 ibdev_err(&rdev->ibdev,
1006                           "Failed to allocate HW AH for Shadow QP");
1007                 goto fail;
1008         }
1009
1010         return ah;
1011
1012 fail:
1013         kfree(ah);
1014         return NULL;
1015 }
1016
1017 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1018                                 (struct bnxt_re_pd *pd,
1019                                  struct bnxt_qplib_res *qp1_res,
1020                                  struct bnxt_qplib_qp *qp1_qp)
1021 {
1022         struct bnxt_re_dev *rdev = pd->rdev;
1023         struct bnxt_re_qp *qp;
1024         int rc;
1025
1026         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1027         if (!qp)
1028                 return NULL;
1029
1030         qp->rdev = rdev;
1031
1032         /* Initialize the shadow QP structure from the QP1 values */
1033         ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1034
1035         qp->qplib_qp.pd = &pd->qplib_pd;
1036         qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1037         qp->qplib_qp.type = IB_QPT_UD;
1038
1039         qp->qplib_qp.max_inline_data = 0;
1040         qp->qplib_qp.sig_type = true;
1041
1042         /* Shadow QP SQ depth should be same as QP1 RQ depth */
1043         qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1044         qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1045         qp->qplib_qp.sq.max_sge = 2;
1046         /* Q full delta can be 1 since it is internal QP */
1047         qp->qplib_qp.sq.q_full_delta = 1;
1048         qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1049         qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1050
1051         qp->qplib_qp.scq = qp1_qp->scq;
1052         qp->qplib_qp.rcq = qp1_qp->rcq;
1053
1054         qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1055         qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1056         qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1057         /* Q full delta can be 1 since it is internal QP */
1058         qp->qplib_qp.rq.q_full_delta = 1;
1059         qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1060         qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1061
1062         qp->qplib_qp.mtu = qp1_qp->mtu;
1063
1064         qp->qplib_qp.sq_hdr_buf_size = 0;
1065         qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1066         qp->qplib_qp.dpi = &rdev->dpi_privileged;
1067
1068         rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1069         if (rc)
1070                 goto fail;
1071
1072         spin_lock_init(&qp->sq_lock);
1073         INIT_LIST_HEAD(&qp->list);
1074         mutex_lock(&rdev->qp_lock);
1075         list_add_tail(&qp->list, &rdev->qp_list);
1076         atomic_inc(&rdev->qp_count);
1077         mutex_unlock(&rdev->qp_lock);
1078         return qp;
1079 fail:
1080         kfree(qp);
1081         return NULL;
1082 }
1083
1084 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1085                                 struct ib_qp_init_attr *init_attr)
1086 {
1087         struct bnxt_qplib_dev_attr *dev_attr;
1088         struct bnxt_qplib_qp *qplqp;
1089         struct bnxt_re_dev *rdev;
1090         struct bnxt_qplib_q *rq;
1091         int entries;
1092
1093         rdev = qp->rdev;
1094         qplqp = &qp->qplib_qp;
1095         rq = &qplqp->rq;
1096         dev_attr = &rdev->dev_attr;
1097
1098         if (init_attr->srq) {
1099                 struct bnxt_re_srq *srq;
1100
1101                 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1102                 qplqp->srq = &srq->qplib_srq;
1103                 rq->max_wqe = 0;
1104         } else {
1105                 rq->max_sge = init_attr->cap.max_recv_sge;
1106                 if (rq->max_sge > dev_attr->max_qp_sges)
1107                         rq->max_sge = dev_attr->max_qp_sges;
1108                 init_attr->cap.max_recv_sge = rq->max_sge;
1109                 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1110                                                        dev_attr->max_qp_sges);
1111                 /* Allocate 1 more than what's provided so posting max doesn't
1112                  * mean empty.
1113                  */
1114                 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
1115                 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1116                 rq->q_full_delta = 0;
1117                 rq->sg_info.pgsize = PAGE_SIZE;
1118                 rq->sg_info.pgshft = PAGE_SHIFT;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1125 {
1126         struct bnxt_qplib_dev_attr *dev_attr;
1127         struct bnxt_qplib_qp *qplqp;
1128         struct bnxt_re_dev *rdev;
1129
1130         rdev = qp->rdev;
1131         qplqp = &qp->qplib_qp;
1132         dev_attr = &rdev->dev_attr;
1133
1134         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1135                 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1136                 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1137                         qplqp->rq.max_sge = dev_attr->max_qp_sges;
1138                 qplqp->rq.max_sge = 6;
1139         }
1140 }
1141
1142 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1143                                 struct ib_qp_init_attr *init_attr,
1144                                 struct ib_udata *udata)
1145 {
1146         struct bnxt_qplib_dev_attr *dev_attr;
1147         struct bnxt_qplib_qp *qplqp;
1148         struct bnxt_re_dev *rdev;
1149         struct bnxt_qplib_q *sq;
1150         int entries;
1151         int diff;
1152         int rc;
1153
1154         rdev = qp->rdev;
1155         qplqp = &qp->qplib_qp;
1156         sq = &qplqp->sq;
1157         dev_attr = &rdev->dev_attr;
1158
1159         sq->max_sge = init_attr->cap.max_send_sge;
1160         if (sq->max_sge > dev_attr->max_qp_sges) {
1161                 sq->max_sge = dev_attr->max_qp_sges;
1162                 init_attr->cap.max_send_sge = sq->max_sge;
1163         }
1164
1165         rc = bnxt_re_setup_swqe_size(qp, init_attr);
1166         if (rc)
1167                 return rc;
1168
1169         entries = init_attr->cap.max_send_wr;
1170         /* Allocate 128 + 1 more than what's provided */
1171         diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1172                 0 : BNXT_QPLIB_RESERVED_QP_WRS;
1173         entries = roundup_pow_of_two(entries + diff + 1);
1174         sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1175         sq->q_full_delta = diff + 1;
1176         /*
1177          * Reserving one slot for Phantom WQE. Application can
1178          * post one extra entry in this case. But allowing this to avoid
1179          * unexpected Queue full condition
1180          */
1181         qplqp->sq.q_full_delta -= 1;
1182         qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1183         qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1184
1185         return 0;
1186 }
1187
1188 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1189                                        struct ib_qp_init_attr *init_attr)
1190 {
1191         struct bnxt_qplib_dev_attr *dev_attr;
1192         struct bnxt_qplib_qp *qplqp;
1193         struct bnxt_re_dev *rdev;
1194         int entries;
1195
1196         rdev = qp->rdev;
1197         qplqp = &qp->qplib_qp;
1198         dev_attr = &rdev->dev_attr;
1199
1200         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1201                 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
1202                 qplqp->sq.max_wqe = min_t(u32, entries,
1203                                           dev_attr->max_qp_wqes + 1);
1204                 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1205                         init_attr->cap.max_send_wr;
1206                 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1207                 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1208                         qplqp->sq.max_sge = dev_attr->max_qp_sges;
1209         }
1210 }
1211
1212 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1213                                 struct ib_qp_init_attr *init_attr)
1214 {
1215         struct bnxt_qplib_chip_ctx *chip_ctx;
1216         int qptype;
1217
1218         chip_ctx = rdev->chip_ctx;
1219
1220         qptype = __from_ib_qp_type(init_attr->qp_type);
1221         if (qptype == IB_QPT_MAX) {
1222                 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1223                 qptype = -EOPNOTSUPP;
1224                 goto out;
1225         }
1226
1227         if (bnxt_qplib_is_chip_gen_p5(chip_ctx) &&
1228             init_attr->qp_type == IB_QPT_GSI)
1229                 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1230 out:
1231         return qptype;
1232 }
1233
1234 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1235                                 struct ib_qp_init_attr *init_attr,
1236                                 struct ib_udata *udata)
1237 {
1238         struct bnxt_qplib_dev_attr *dev_attr;
1239         struct bnxt_qplib_qp *qplqp;
1240         struct bnxt_re_dev *rdev;
1241         struct bnxt_re_cq *cq;
1242         int rc = 0, qptype;
1243
1244         rdev = qp->rdev;
1245         qplqp = &qp->qplib_qp;
1246         dev_attr = &rdev->dev_attr;
1247
1248         /* Setup misc params */
1249         ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1250         qplqp->pd = &pd->qplib_pd;
1251         qplqp->qp_handle = (u64)qplqp;
1252         qplqp->max_inline_data = init_attr->cap.max_inline_data;
1253         qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
1254                             true : false);
1255         qptype = bnxt_re_init_qp_type(rdev, init_attr);
1256         if (qptype < 0) {
1257                 rc = qptype;
1258                 goto out;
1259         }
1260         qplqp->type = (u8)qptype;
1261         qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
1262
1263         if (init_attr->qp_type == IB_QPT_RC) {
1264                 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1265                 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1266         }
1267         qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1268         qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1269         if (init_attr->create_flags) {
1270                 ibdev_dbg(&rdev->ibdev,
1271                           "QP create flags 0x%x not supported",
1272                           init_attr->create_flags);
1273                 return -EOPNOTSUPP;
1274         }
1275
1276         /* Setup CQs */
1277         if (init_attr->send_cq) {
1278                 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1279                 qplqp->scq = &cq->qplib_cq;
1280                 qp->scq = cq;
1281         }
1282
1283         if (init_attr->recv_cq) {
1284                 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1285                 qplqp->rcq = &cq->qplib_cq;
1286                 qp->rcq = cq;
1287         }
1288
1289         /* Setup RQ/SRQ */
1290         rc = bnxt_re_init_rq_attr(qp, init_attr);
1291         if (rc)
1292                 goto out;
1293         if (init_attr->qp_type == IB_QPT_GSI)
1294                 bnxt_re_adjust_gsi_rq_attr(qp);
1295
1296         /* Setup SQ */
1297         rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
1298         if (rc)
1299                 goto out;
1300         if (init_attr->qp_type == IB_QPT_GSI)
1301                 bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
1302
1303         if (udata) /* This will update DPI and qp_handle */
1304                 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1305 out:
1306         return rc;
1307 }
1308
1309 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1310                                      struct bnxt_re_pd *pd)
1311 {
1312         struct bnxt_re_sqp_entries *sqp_tbl = NULL;
1313         struct bnxt_re_dev *rdev;
1314         struct bnxt_re_qp *sqp;
1315         struct bnxt_re_ah *sah;
1316         int rc = 0;
1317
1318         rdev = qp->rdev;
1319         /* Create a shadow QP to handle the QP1 traffic */
1320         sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES,
1321                           GFP_KERNEL);
1322         if (!sqp_tbl)
1323                 return -ENOMEM;
1324         rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1325
1326         sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1327         if (!sqp) {
1328                 rc = -ENODEV;
1329                 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1330                 goto out;
1331         }
1332         rdev->gsi_ctx.gsi_sqp = sqp;
1333
1334         sqp->rcq = qp->rcq;
1335         sqp->scq = qp->scq;
1336         sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1337                                           &qp->qplib_qp);
1338         if (!sah) {
1339                 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1340                                       &sqp->qplib_qp);
1341                 rc = -ENODEV;
1342                 ibdev_err(&rdev->ibdev,
1343                           "Failed to create AH entry for ShadowQP");
1344                 goto out;
1345         }
1346         rdev->gsi_ctx.gsi_sah = sah;
1347
1348         return 0;
1349 out:
1350         kfree(sqp_tbl);
1351         return rc;
1352 }
1353
1354 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1355                                  struct ib_qp_init_attr *init_attr)
1356 {
1357         struct bnxt_re_dev *rdev;
1358         struct bnxt_qplib_qp *qplqp;
1359         int rc = 0;
1360
1361         rdev = qp->rdev;
1362         qplqp = &qp->qplib_qp;
1363
1364         qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1365         qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1366
1367         rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1368         if (rc) {
1369                 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1370                 goto out;
1371         }
1372
1373         rc = bnxt_re_create_shadow_gsi(qp, pd);
1374 out:
1375         return rc;
1376 }
1377
1378 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1379                                    struct ib_qp_init_attr *init_attr,
1380                                    struct bnxt_qplib_dev_attr *dev_attr)
1381 {
1382         bool rc = true;
1383
1384         if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1385             init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1386             init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1387             init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1388             init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1389                 ibdev_err(&rdev->ibdev,
1390                           "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1391                           init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1392                           init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1393                           init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1394                           init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1395                           init_attr->cap.max_inline_data,
1396                           dev_attr->max_inline_data);
1397                 rc = false;
1398         }
1399         return rc;
1400 }
1401
1402 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1403                       struct ib_udata *udata)
1404 {
1405         struct ib_pd *ib_pd = ib_qp->pd;
1406         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1407         struct bnxt_re_dev *rdev = pd->rdev;
1408         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1409         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1410         int rc;
1411
1412         rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1413         if (!rc) {
1414                 rc = -EINVAL;
1415                 goto fail;
1416         }
1417
1418         qp->rdev = rdev;
1419         rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
1420         if (rc)
1421                 goto fail;
1422
1423         if (qp_init_attr->qp_type == IB_QPT_GSI &&
1424             !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) {
1425                 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1426                 if (rc == -ENODEV)
1427                         goto qp_destroy;
1428                 if (rc)
1429                         goto fail;
1430         } else {
1431                 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1432                 if (rc) {
1433                         ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1434                         goto free_umem;
1435                 }
1436                 if (udata) {
1437                         struct bnxt_re_qp_resp resp;
1438
1439                         resp.qpid = qp->qplib_qp.id;
1440                         resp.rsvd = 0;
1441                         rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1442                         if (rc) {
1443                                 ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
1444                                 goto qp_destroy;
1445                         }
1446                 }
1447         }
1448
1449         qp->ib_qp.qp_num = qp->qplib_qp.id;
1450         if (qp_init_attr->qp_type == IB_QPT_GSI)
1451                 rdev->gsi_ctx.gsi_qp = qp;
1452         spin_lock_init(&qp->sq_lock);
1453         spin_lock_init(&qp->rq_lock);
1454         INIT_LIST_HEAD(&qp->list);
1455         mutex_lock(&rdev->qp_lock);
1456         list_add_tail(&qp->list, &rdev->qp_list);
1457         mutex_unlock(&rdev->qp_lock);
1458         atomic_inc(&rdev->qp_count);
1459
1460         return 0;
1461 qp_destroy:
1462         bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1463 free_umem:
1464         ib_umem_release(qp->rumem);
1465         ib_umem_release(qp->sumem);
1466 fail:
1467         return rc;
1468 }
1469
1470 static u8 __from_ib_qp_state(enum ib_qp_state state)
1471 {
1472         switch (state) {
1473         case IB_QPS_RESET:
1474                 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1475         case IB_QPS_INIT:
1476                 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1477         case IB_QPS_RTR:
1478                 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1479         case IB_QPS_RTS:
1480                 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1481         case IB_QPS_SQD:
1482                 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1483         case IB_QPS_SQE:
1484                 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1485         case IB_QPS_ERR:
1486         default:
1487                 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1488         }
1489 }
1490
1491 static enum ib_qp_state __to_ib_qp_state(u8 state)
1492 {
1493         switch (state) {
1494         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1495                 return IB_QPS_RESET;
1496         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1497                 return IB_QPS_INIT;
1498         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1499                 return IB_QPS_RTR;
1500         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1501                 return IB_QPS_RTS;
1502         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1503                 return IB_QPS_SQD;
1504         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1505                 return IB_QPS_SQE;
1506         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1507         default:
1508                 return IB_QPS_ERR;
1509         }
1510 }
1511
1512 static u32 __from_ib_mtu(enum ib_mtu mtu)
1513 {
1514         switch (mtu) {
1515         case IB_MTU_256:
1516                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1517         case IB_MTU_512:
1518                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1519         case IB_MTU_1024:
1520                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1521         case IB_MTU_2048:
1522                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1523         case IB_MTU_4096:
1524                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1525         default:
1526                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1527         }
1528 }
1529
1530 static enum ib_mtu __to_ib_mtu(u32 mtu)
1531 {
1532         switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1533         case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1534                 return IB_MTU_256;
1535         case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1536                 return IB_MTU_512;
1537         case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1538                 return IB_MTU_1024;
1539         case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1540                 return IB_MTU_2048;
1541         case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1542                 return IB_MTU_4096;
1543         default:
1544                 return IB_MTU_2048;
1545         }
1546 }
1547
1548 /* Shared Receive Queues */
1549 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1550 {
1551         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1552                                                ib_srq);
1553         struct bnxt_re_dev *rdev = srq->rdev;
1554         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1555         struct bnxt_qplib_nq *nq = NULL;
1556
1557         if (qplib_srq->cq)
1558                 nq = qplib_srq->cq->nq;
1559         bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1560         ib_umem_release(srq->umem);
1561         atomic_dec(&rdev->srq_count);
1562         if (nq)
1563                 nq->budget--;
1564         return 0;
1565 }
1566
1567 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1568                                  struct bnxt_re_pd *pd,
1569                                  struct bnxt_re_srq *srq,
1570                                  struct ib_udata *udata)
1571 {
1572         struct bnxt_re_srq_req ureq;
1573         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1574         struct ib_umem *umem;
1575         int bytes = 0;
1576         struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1577                 udata, struct bnxt_re_ucontext, ib_uctx);
1578
1579         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1580                 return -EFAULT;
1581
1582         bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1583         bytes = PAGE_ALIGN(bytes);
1584         umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1585                            IB_ACCESS_LOCAL_WRITE);
1586         if (IS_ERR(umem))
1587                 return PTR_ERR(umem);
1588
1589         srq->umem = umem;
1590         qplib_srq->sg_info.umem = umem;
1591         qplib_srq->sg_info.pgsize = PAGE_SIZE;
1592         qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1593         qplib_srq->srq_handle = ureq.srq_handle;
1594         qplib_srq->dpi = &cntx->dpi;
1595
1596         return 0;
1597 }
1598
1599 int bnxt_re_create_srq(struct ib_srq *ib_srq,
1600                        struct ib_srq_init_attr *srq_init_attr,
1601                        struct ib_udata *udata)
1602 {
1603         struct bnxt_qplib_dev_attr *dev_attr;
1604         struct bnxt_qplib_nq *nq = NULL;
1605         struct bnxt_re_dev *rdev;
1606         struct bnxt_re_srq *srq;
1607         struct bnxt_re_pd *pd;
1608         struct ib_pd *ib_pd;
1609         int rc, entries;
1610
1611         ib_pd = ib_srq->pd;
1612         pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1613         rdev = pd->rdev;
1614         dev_attr = &rdev->dev_attr;
1615         srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1616
1617         if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1618                 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
1619                 rc = -EINVAL;
1620                 goto exit;
1621         }
1622
1623         if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1624                 rc = -EOPNOTSUPP;
1625                 goto exit;
1626         }
1627
1628         srq->rdev = rdev;
1629         srq->qplib_srq.pd = &pd->qplib_pd;
1630         srq->qplib_srq.dpi = &rdev->dpi_privileged;
1631         /* Allocate 1 more than what's provided so posting max doesn't
1632          * mean empty
1633          */
1634         entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
1635         if (entries > dev_attr->max_srq_wqes + 1)
1636                 entries = dev_attr->max_srq_wqes + 1;
1637         srq->qplib_srq.max_wqe = entries;
1638
1639         srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1640          /* 128 byte wqe size for SRQ . So use max sges */
1641         srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
1642         srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1643         srq->srq_limit = srq_init_attr->attr.srq_limit;
1644         srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
1645         nq = &rdev->nq[0];
1646
1647         if (udata) {
1648                 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1649                 if (rc)
1650                         goto fail;
1651         }
1652
1653         rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1654         if (rc) {
1655                 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
1656                 goto fail;
1657         }
1658
1659         if (udata) {
1660                 struct bnxt_re_srq_resp resp;
1661
1662                 resp.srqid = srq->qplib_srq.id;
1663                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1664                 if (rc) {
1665                         ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
1666                         bnxt_qplib_destroy_srq(&rdev->qplib_res,
1667                                                &srq->qplib_srq);
1668                         goto fail;
1669                 }
1670         }
1671         if (nq)
1672                 nq->budget++;
1673         atomic_inc(&rdev->srq_count);
1674
1675         return 0;
1676
1677 fail:
1678         ib_umem_release(srq->umem);
1679 exit:
1680         return rc;
1681 }
1682
1683 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1684                        enum ib_srq_attr_mask srq_attr_mask,
1685                        struct ib_udata *udata)
1686 {
1687         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1688                                                ib_srq);
1689         struct bnxt_re_dev *rdev = srq->rdev;
1690         int rc;
1691
1692         switch (srq_attr_mask) {
1693         case IB_SRQ_MAX_WR:
1694                 /* SRQ resize is not supported */
1695                 break;
1696         case IB_SRQ_LIMIT:
1697                 /* Change the SRQ threshold */
1698                 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1699                         return -EINVAL;
1700
1701                 srq->qplib_srq.threshold = srq_attr->srq_limit;
1702                 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1703                 if (rc) {
1704                         ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!");
1705                         return rc;
1706                 }
1707                 /* On success, update the shadow */
1708                 srq->srq_limit = srq_attr->srq_limit;
1709                 /* No need to Build and send response back to udata */
1710                 break;
1711         default:
1712                 ibdev_err(&rdev->ibdev,
1713                           "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1714                 return -EINVAL;
1715         }
1716         return 0;
1717 }
1718
1719 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1720 {
1721         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1722                                                ib_srq);
1723         struct bnxt_re_srq tsrq;
1724         struct bnxt_re_dev *rdev = srq->rdev;
1725         int rc;
1726
1727         /* Get live SRQ attr */
1728         tsrq.qplib_srq.id = srq->qplib_srq.id;
1729         rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
1730         if (rc) {
1731                 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
1732                 return rc;
1733         }
1734         srq_attr->max_wr = srq->qplib_srq.max_wqe;
1735         srq_attr->max_sge = srq->qplib_srq.max_sge;
1736         srq_attr->srq_limit = tsrq.qplib_srq.threshold;
1737
1738         return 0;
1739 }
1740
1741 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1742                           const struct ib_recv_wr **bad_wr)
1743 {
1744         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1745                                                ib_srq);
1746         struct bnxt_qplib_swqe wqe;
1747         unsigned long flags;
1748         int rc = 0;
1749
1750         spin_lock_irqsave(&srq->lock, flags);
1751         while (wr) {
1752                 /* Transcribe each ib_recv_wr to qplib_swqe */
1753                 wqe.num_sge = wr->num_sge;
1754                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
1755                 wqe.wr_id = wr->wr_id;
1756                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1757
1758                 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1759                 if (rc) {
1760                         *bad_wr = wr;
1761                         break;
1762                 }
1763                 wr = wr->next;
1764         }
1765         spin_unlock_irqrestore(&srq->lock, flags);
1766
1767         return rc;
1768 }
1769 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1770                                     struct bnxt_re_qp *qp1_qp,
1771                                     int qp_attr_mask)
1772 {
1773         struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
1774         int rc = 0;
1775
1776         if (qp_attr_mask & IB_QP_STATE) {
1777                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1778                 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1779         }
1780         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1781                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1782                 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1783         }
1784
1785         if (qp_attr_mask & IB_QP_QKEY) {
1786                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1787                 /* Using a Random  QKEY */
1788                 qp->qplib_qp.qkey = 0x81818181;
1789         }
1790         if (qp_attr_mask & IB_QP_SQ_PSN) {
1791                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1792                 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1793         }
1794
1795         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1796         if (rc)
1797                 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
1798         return rc;
1799 }
1800
1801 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1802                       int qp_attr_mask, struct ib_udata *udata)
1803 {
1804         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1805         struct bnxt_re_dev *rdev = qp->rdev;
1806         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1807         enum ib_qp_state curr_qp_state, new_qp_state;
1808         int rc, entries;
1809         unsigned int flags;
1810         u8 nw_type;
1811
1812         if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1813                 return -EOPNOTSUPP;
1814
1815         qp->qplib_qp.modify_flags = 0;
1816         if (qp_attr_mask & IB_QP_STATE) {
1817                 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1818                 new_qp_state = qp_attr->qp_state;
1819                 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1820                                         ib_qp->qp_type, qp_attr_mask)) {
1821                         ibdev_err(&rdev->ibdev,
1822                                   "Invalid attribute mask: %#x specified ",
1823                                   qp_attr_mask);
1824                         ibdev_err(&rdev->ibdev,
1825                                   "for qpn: %#x type: %#x",
1826                                   ib_qp->qp_num, ib_qp->qp_type);
1827                         ibdev_err(&rdev->ibdev,
1828                                   "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1829                                   curr_qp_state, new_qp_state);
1830                         return -EINVAL;
1831                 }
1832                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1833                 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1834
1835                 if (!qp->sumem &&
1836                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1837                         ibdev_dbg(&rdev->ibdev,
1838                                   "Move QP = %p to flush list\n", qp);
1839                         flags = bnxt_re_lock_cqs(qp);
1840                         bnxt_qplib_add_flush_qp(&qp->qplib_qp);
1841                         bnxt_re_unlock_cqs(qp, flags);
1842                 }
1843                 if (!qp->sumem &&
1844                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1845                         ibdev_dbg(&rdev->ibdev,
1846                                   "Move QP = %p out of flush list\n", qp);
1847                         flags = bnxt_re_lock_cqs(qp);
1848                         bnxt_qplib_clean_qp(&qp->qplib_qp);
1849                         bnxt_re_unlock_cqs(qp, flags);
1850                 }
1851         }
1852         if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1853                 qp->qplib_qp.modify_flags |=
1854                                 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1855                 qp->qplib_qp.en_sqd_async_notify = true;
1856         }
1857         if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1858                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1859                 qp->qplib_qp.access =
1860                         __from_ib_access_flags(qp_attr->qp_access_flags);
1861                 /* LOCAL_WRITE access must be set to allow RC receive */
1862                 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1863                 /* Temp: Set all params on QP as of now */
1864                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
1865                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
1866         }
1867         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1868                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1869                 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1870         }
1871         if (qp_attr_mask & IB_QP_QKEY) {
1872                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1873                 qp->qplib_qp.qkey = qp_attr->qkey;
1874         }
1875         if (qp_attr_mask & IB_QP_AV) {
1876                 const struct ib_global_route *grh =
1877                         rdma_ah_read_grh(&qp_attr->ah_attr);
1878                 const struct ib_gid_attr *sgid_attr;
1879                 struct bnxt_re_gid_ctx *ctx;
1880
1881                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1882                                      CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1883                                      CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1884                                      CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1885                                      CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1886                                      CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1887                                      CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1888                 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1889                        sizeof(qp->qplib_qp.ah.dgid.data));
1890                 qp->qplib_qp.ah.flow_label = grh->flow_label;
1891                 sgid_attr = grh->sgid_attr;
1892                 /* Get the HW context of the GID. The reference
1893                  * of GID table entry is already taken by the caller.
1894                  */
1895                 ctx = rdma_read_gid_hw_context(sgid_attr);
1896                 qp->qplib_qp.ah.sgid_index = ctx->idx;
1897                 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
1898                 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
1899                 qp->qplib_qp.ah.traffic_class = grh->traffic_class;
1900                 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
1901                 ether_addr_copy(qp->qplib_qp.ah.dmac,
1902                                 qp_attr->ah_attr.roce.dmac);
1903
1904                 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
1905                                              &qp->qplib_qp.smac[0]);
1906                 if (rc)
1907                         return rc;
1908
1909                 nw_type = rdma_gid_attr_network_type(sgid_attr);
1910                 switch (nw_type) {
1911                 case RDMA_NETWORK_IPV4:
1912                         qp->qplib_qp.nw_type =
1913                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1914                         break;
1915                 case RDMA_NETWORK_IPV6:
1916                         qp->qplib_qp.nw_type =
1917                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1918                         break;
1919                 default:
1920                         qp->qplib_qp.nw_type =
1921                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1922                         break;
1923                 }
1924         }
1925
1926         if (qp_attr_mask & IB_QP_PATH_MTU) {
1927                 qp->qplib_qp.modify_flags |=
1928                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1929                 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1930                 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
1931         } else if (qp_attr->qp_state == IB_QPS_RTR) {
1932                 qp->qplib_qp.modify_flags |=
1933                         CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1934                 qp->qplib_qp.path_mtu =
1935                         __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1936                 qp->qplib_qp.mtu =
1937                         ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1938         }
1939
1940         if (qp_attr_mask & IB_QP_TIMEOUT) {
1941                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1942                 qp->qplib_qp.timeout = qp_attr->timeout;
1943         }
1944         if (qp_attr_mask & IB_QP_RETRY_CNT) {
1945                 qp->qplib_qp.modify_flags |=
1946                                 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1947                 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1948         }
1949         if (qp_attr_mask & IB_QP_RNR_RETRY) {
1950                 qp->qplib_qp.modify_flags |=
1951                                 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1952                 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1953         }
1954         if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1955                 qp->qplib_qp.modify_flags |=
1956                                 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1957                 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1958         }
1959         if (qp_attr_mask & IB_QP_RQ_PSN) {
1960                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1961                 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1962         }
1963         if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1964                 qp->qplib_qp.modify_flags |=
1965                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1966                 /* Cap the max_rd_atomic to device max */
1967                 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
1968                                                    dev_attr->max_qp_rd_atom);
1969         }
1970         if (qp_attr_mask & IB_QP_SQ_PSN) {
1971                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1972                 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1973         }
1974         if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1975                 if (qp_attr->max_dest_rd_atomic >
1976                     dev_attr->max_qp_init_rd_atom) {
1977                         ibdev_err(&rdev->ibdev,
1978                                   "max_dest_rd_atomic requested%d is > dev_max%d",
1979                                   qp_attr->max_dest_rd_atomic,
1980                                   dev_attr->max_qp_init_rd_atom);
1981                         return -EINVAL;
1982                 }
1983
1984                 qp->qplib_qp.modify_flags |=
1985                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1986                 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1987         }
1988         if (qp_attr_mask & IB_QP_CAP) {
1989                 qp->qplib_qp.modify_flags |=
1990                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1991                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1992                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1993                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1994                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1995                 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1996                     (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1997                     (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1998                     (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
1999                     (qp_attr->cap.max_inline_data >=
2000                                                 dev_attr->max_inline_data)) {
2001                         ibdev_err(&rdev->ibdev,
2002                                   "Create QP failed - max exceeded");
2003                         return -EINVAL;
2004                 }
2005                 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
2006                 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2007                                                 dev_attr->max_qp_wqes + 1);
2008                 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2009                                                 qp_attr->cap.max_send_wr;
2010                 /*
2011                  * Reserving one slot for Phantom WQE. Some application can
2012                  * post one extra entry in this case. Allowing this to avoid
2013                  * unexpected Queue full condition
2014                  */
2015                 qp->qplib_qp.sq.q_full_delta -= 1;
2016                 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2017                 if (qp->qplib_qp.rq.max_wqe) {
2018                         entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
2019                         qp->qplib_qp.rq.max_wqe =
2020                                 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2021                         qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2022                                                        qp_attr->cap.max_recv_wr;
2023                         qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2024                 } else {
2025                         /* SRQ was used prior, just ignore the RQ caps */
2026                 }
2027         }
2028         if (qp_attr_mask & IB_QP_DEST_QPN) {
2029                 qp->qplib_qp.modify_flags |=
2030                                 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2031                 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2032         }
2033         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2034         if (rc) {
2035                 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2036                 return rc;
2037         }
2038         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2039                 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2040         return rc;
2041 }
2042
2043 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2044                      int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2045 {
2046         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2047         struct bnxt_re_dev *rdev = qp->rdev;
2048         struct bnxt_qplib_qp *qplib_qp;
2049         int rc;
2050
2051         qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2052         if (!qplib_qp)
2053                 return -ENOMEM;
2054
2055         qplib_qp->id = qp->qplib_qp.id;
2056         qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2057
2058         rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2059         if (rc) {
2060                 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2061                 goto out;
2062         }
2063         qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2064         qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2065         qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2066         qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
2067         qp_attr->pkey_index = qplib_qp->pkey_index;
2068         qp_attr->qkey = qplib_qp->qkey;
2069         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2070         rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
2071                         qplib_qp->ah.host_sgid_index,
2072                         qplib_qp->ah.hop_limit,
2073                         qplib_qp->ah.traffic_class);
2074         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2075         rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2076         ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2077         qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2078         qp_attr->timeout = qplib_qp->timeout;
2079         qp_attr->retry_cnt = qplib_qp->retry_cnt;
2080         qp_attr->rnr_retry = qplib_qp->rnr_retry;
2081         qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2082         qp_attr->rq_psn = qplib_qp->rq.psn;
2083         qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2084         qp_attr->sq_psn = qplib_qp->sq.psn;
2085         qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2086         qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2087                                                          IB_SIGNAL_REQ_WR;
2088         qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2089
2090         qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2091         qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2092         qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2093         qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2094         qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2095         qp_init_attr->cap = qp_attr->cap;
2096
2097 out:
2098         kfree(qplib_qp);
2099         return rc;
2100 }
2101
2102 /* Routine for sending QP1 packets for RoCE V1 an V2
2103  */
2104 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2105                                      const struct ib_send_wr *wr,
2106                                      struct bnxt_qplib_swqe *wqe,
2107                                      int payload_size)
2108 {
2109         struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2110                                              ib_ah);
2111         struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2112         const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2113         struct bnxt_qplib_sge sge;
2114         u8 nw_type;
2115         u16 ether_type;
2116         union ib_gid dgid;
2117         bool is_eth = false;
2118         bool is_vlan = false;
2119         bool is_grh = false;
2120         bool is_udp = false;
2121         u8 ip_version = 0;
2122         u16 vlan_id = 0xFFFF;
2123         void *buf;
2124         int i, rc = 0;
2125
2126         memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2127
2128         rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2129         if (rc)
2130                 return rc;
2131
2132         /* Get network header type for this GID */
2133         nw_type = rdma_gid_attr_network_type(sgid_attr);
2134         switch (nw_type) {
2135         case RDMA_NETWORK_IPV4:
2136                 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2137                 break;
2138         case RDMA_NETWORK_IPV6:
2139                 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2140                 break;
2141         default:
2142                 nw_type = BNXT_RE_ROCE_V1_PACKET;
2143                 break;
2144         }
2145         memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2146         is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2147         if (is_udp) {
2148                 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2149                         ip_version = 4;
2150                         ether_type = ETH_P_IP;
2151                 } else {
2152                         ip_version = 6;
2153                         ether_type = ETH_P_IPV6;
2154                 }
2155                 is_grh = false;
2156         } else {
2157                 ether_type = ETH_P_IBOE;
2158                 is_grh = true;
2159         }
2160
2161         is_eth = true;
2162         is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
2163
2164         ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2165                           ip_version, is_udp, 0, &qp->qp1_hdr);
2166
2167         /* ETH */
2168         ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2169         ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2170
2171         /* For vlan, check the sgid for vlan existence */
2172
2173         if (!is_vlan) {
2174                 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2175         } else {
2176                 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2177                 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2178         }
2179
2180         if (is_grh || (ip_version == 6)) {
2181                 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2182                        sizeof(sgid_attr->gid));
2183                 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2184                        sizeof(sgid_attr->gid));
2185                 qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
2186         }
2187
2188         if (ip_version == 4) {
2189                 qp->qp1_hdr.ip4.tos = 0;
2190                 qp->qp1_hdr.ip4.id = 0;
2191                 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2192                 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2193
2194                 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2195                 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2196                 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2197         }
2198
2199         if (is_udp) {
2200                 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2201                 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2202                 qp->qp1_hdr.udp.csum = 0;
2203         }
2204
2205         /* BTH */
2206         if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2207                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2208                 qp->qp1_hdr.immediate_present = 1;
2209         } else {
2210                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2211         }
2212         if (wr->send_flags & IB_SEND_SOLICITED)
2213                 qp->qp1_hdr.bth.solicited_event = 1;
2214         /* pad_count */
2215         qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2216
2217         /* P_key for QP1 is for all members */
2218         qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2219         qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2220         qp->qp1_hdr.bth.ack_req = 0;
2221         qp->send_psn++;
2222         qp->send_psn &= BTH_PSN_MASK;
2223         qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2224         /* DETH */
2225         /* Use the priviledged Q_Key for QP1 */
2226         qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2227         qp->qp1_hdr.deth.source_qpn = IB_QP1;
2228
2229         /* Pack the QP1 to the transmit buffer */
2230         buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2231         if (buf) {
2232                 ib_ud_header_pack(&qp->qp1_hdr, buf);
2233                 for (i = wqe->num_sge; i; i--) {
2234                         wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2235                         wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2236                         wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2237                 }
2238
2239                 /*
2240                  * Max Header buf size for IPV6 RoCE V2 is 86,
2241                  * which is same as the QP1 SQ header buffer.
2242                  * Header buf size for IPV4 RoCE V2 can be 66.
2243                  * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2244                  * Subtract 20 bytes from QP1 SQ header buf size
2245                  */
2246                 if (is_udp && ip_version == 4)
2247                         sge.size -= 20;
2248                 /*
2249                  * Max Header buf size for RoCE V1 is 78.
2250                  * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2251                  * Subtract 8 bytes from QP1 SQ header buf size
2252                  */
2253                 if (!is_udp)
2254                         sge.size -= 8;
2255
2256                 /* Subtract 4 bytes for non vlan packets */
2257                 if (!is_vlan)
2258                         sge.size -= 4;
2259
2260                 wqe->sg_list[0].addr = sge.addr;
2261                 wqe->sg_list[0].lkey = sge.lkey;
2262                 wqe->sg_list[0].size = sge.size;
2263                 wqe->num_sge++;
2264
2265         } else {
2266                 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2267                 rc = -ENOMEM;
2268         }
2269         return rc;
2270 }
2271
2272 /* For the MAD layer, it only provides the recv SGE the size of
2273  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
2274  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
2275  * receive packet (334 bytes) with no VLAN and then copy the GRH
2276  * and the MAD datagram out to the provided SGE.
2277  */
2278 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2279                                             const struct ib_recv_wr *wr,
2280                                             struct bnxt_qplib_swqe *wqe,
2281                                             int payload_size)
2282 {
2283         struct bnxt_re_sqp_entries *sqp_entry;
2284         struct bnxt_qplib_sge ref, sge;
2285         struct bnxt_re_dev *rdev;
2286         u32 rq_prod_index;
2287
2288         rdev = qp->rdev;
2289
2290         rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2291
2292         if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2293                 return -ENOMEM;
2294
2295         /* Create 1 SGE to receive the entire
2296          * ethernet packet
2297          */
2298         /* Save the reference from ULP */
2299         ref.addr = wqe->sg_list[0].addr;
2300         ref.lkey = wqe->sg_list[0].lkey;
2301         ref.size = wqe->sg_list[0].size;
2302
2303         sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2304
2305         /* SGE 1 */
2306         wqe->sg_list[0].addr = sge.addr;
2307         wqe->sg_list[0].lkey = sge.lkey;
2308         wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2309         sge.size -= wqe->sg_list[0].size;
2310
2311         sqp_entry->sge.addr = ref.addr;
2312         sqp_entry->sge.lkey = ref.lkey;
2313         sqp_entry->sge.size = ref.size;
2314         /* Store the wrid for reporting completion */
2315         sqp_entry->wrid = wqe->wr_id;
2316         /* change the wqe->wrid to table index */
2317         wqe->wr_id = rq_prod_index;
2318         return 0;
2319 }
2320
2321 static int is_ud_qp(struct bnxt_re_qp *qp)
2322 {
2323         return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2324                 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2325 }
2326
2327 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2328                                   const struct ib_send_wr *wr,
2329                                   struct bnxt_qplib_swqe *wqe)
2330 {
2331         struct bnxt_re_ah *ah = NULL;
2332
2333         if (is_ud_qp(qp)) {
2334                 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2335                 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2336                 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2337                 wqe->send.avid = ah->qplib_ah.id;
2338         }
2339         switch (wr->opcode) {
2340         case IB_WR_SEND:
2341                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2342                 break;
2343         case IB_WR_SEND_WITH_IMM:
2344                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2345                 wqe->send.imm_data = wr->ex.imm_data;
2346                 break;
2347         case IB_WR_SEND_WITH_INV:
2348                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2349                 wqe->send.inv_key = wr->ex.invalidate_rkey;
2350                 break;
2351         default:
2352                 return -EINVAL;
2353         }
2354         if (wr->send_flags & IB_SEND_SIGNALED)
2355                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2356         if (wr->send_flags & IB_SEND_FENCE)
2357                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2358         if (wr->send_flags & IB_SEND_SOLICITED)
2359                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2360         if (wr->send_flags & IB_SEND_INLINE)
2361                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2362
2363         return 0;
2364 }
2365
2366 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2367                                   struct bnxt_qplib_swqe *wqe)
2368 {
2369         switch (wr->opcode) {
2370         case IB_WR_RDMA_WRITE:
2371                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2372                 break;
2373         case IB_WR_RDMA_WRITE_WITH_IMM:
2374                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2375                 wqe->rdma.imm_data = wr->ex.imm_data;
2376                 break;
2377         case IB_WR_RDMA_READ:
2378                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2379                 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2380                 break;
2381         default:
2382                 return -EINVAL;
2383         }
2384         wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2385         wqe->rdma.r_key = rdma_wr(wr)->rkey;
2386         if (wr->send_flags & IB_SEND_SIGNALED)
2387                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2388         if (wr->send_flags & IB_SEND_FENCE)
2389                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2390         if (wr->send_flags & IB_SEND_SOLICITED)
2391                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2392         if (wr->send_flags & IB_SEND_INLINE)
2393                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2394
2395         return 0;
2396 }
2397
2398 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2399                                     struct bnxt_qplib_swqe *wqe)
2400 {
2401         switch (wr->opcode) {
2402         case IB_WR_ATOMIC_CMP_AND_SWP:
2403                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2404                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2405                 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2406                 break;
2407         case IB_WR_ATOMIC_FETCH_AND_ADD:
2408                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2409                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2410                 break;
2411         default:
2412                 return -EINVAL;
2413         }
2414         wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2415         wqe->atomic.r_key = atomic_wr(wr)->rkey;
2416         if (wr->send_flags & IB_SEND_SIGNALED)
2417                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2418         if (wr->send_flags & IB_SEND_FENCE)
2419                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2420         if (wr->send_flags & IB_SEND_SOLICITED)
2421                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2422         return 0;
2423 }
2424
2425 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2426                                  struct bnxt_qplib_swqe *wqe)
2427 {
2428         wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2429         wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2430
2431         /* Need unconditional fence for local invalidate
2432          * opcode to work as expected.
2433          */
2434         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2435
2436         if (wr->send_flags & IB_SEND_SIGNALED)
2437                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2438         if (wr->send_flags & IB_SEND_SOLICITED)
2439                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2440
2441         return 0;
2442 }
2443
2444 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2445                                  struct bnxt_qplib_swqe *wqe)
2446 {
2447         struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2448         struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2449         int access = wr->access;
2450
2451         wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2452         wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2453         wqe->frmr.page_list = mr->pages;
2454         wqe->frmr.page_list_len = mr->npages;
2455         wqe->frmr.levels = qplib_frpl->hwq.level;
2456         wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2457
2458         /* Need unconditional fence for reg_mr
2459          * opcode to function as expected.
2460          */
2461
2462         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2463
2464         if (wr->wr.send_flags & IB_SEND_SIGNALED)
2465                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2466
2467         if (access & IB_ACCESS_LOCAL_WRITE)
2468                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2469         if (access & IB_ACCESS_REMOTE_READ)
2470                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2471         if (access & IB_ACCESS_REMOTE_WRITE)
2472                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2473         if (access & IB_ACCESS_REMOTE_ATOMIC)
2474                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2475         if (access & IB_ACCESS_MW_BIND)
2476                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2477
2478         wqe->frmr.l_key = wr->key;
2479         wqe->frmr.length = wr->mr->length;
2480         wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
2481         wqe->frmr.va = wr->mr->iova;
2482         return 0;
2483 }
2484
2485 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2486                                     const struct ib_send_wr *wr,
2487                                     struct bnxt_qplib_swqe *wqe)
2488 {
2489         /*  Copy the inline data to the data  field */
2490         u8 *in_data;
2491         u32 i, sge_len;
2492         void *sge_addr;
2493
2494         in_data = wqe->inline_data;
2495         for (i = 0; i < wr->num_sge; i++) {
2496                 sge_addr = (void *)(unsigned long)
2497                                 wr->sg_list[i].addr;
2498                 sge_len = wr->sg_list[i].length;
2499
2500                 if ((sge_len + wqe->inline_len) >
2501                     BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2502                         ibdev_err(&rdev->ibdev,
2503                                   "Inline data size requested > supported value");
2504                         return -EINVAL;
2505                 }
2506                 sge_len = wr->sg_list[i].length;
2507
2508                 memcpy(in_data, sge_addr, sge_len);
2509                 in_data += wr->sg_list[i].length;
2510                 wqe->inline_len += wr->sg_list[i].length;
2511         }
2512         return wqe->inline_len;
2513 }
2514
2515 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2516                                    const struct ib_send_wr *wr,
2517                                    struct bnxt_qplib_swqe *wqe)
2518 {
2519         int payload_sz = 0;
2520
2521         if (wr->send_flags & IB_SEND_INLINE)
2522                 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2523         else
2524                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2525                                                wqe->num_sge);
2526
2527         return payload_sz;
2528 }
2529
2530 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2531 {
2532         if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2533              qp->ib_qp.qp_type == IB_QPT_GSI ||
2534              qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2535              qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2536                 int qp_attr_mask;
2537                 struct ib_qp_attr qp_attr;
2538
2539                 qp_attr_mask = IB_QP_STATE;
2540                 qp_attr.qp_state = IB_QPS_RTS;
2541                 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2542                 qp->qplib_qp.wqe_cnt = 0;
2543         }
2544 }
2545
2546 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2547                                        struct bnxt_re_qp *qp,
2548                                        const struct ib_send_wr *wr)
2549 {
2550         int rc = 0, payload_sz = 0;
2551         unsigned long flags;
2552
2553         spin_lock_irqsave(&qp->sq_lock, flags);
2554         while (wr) {
2555                 struct bnxt_qplib_swqe wqe = {};
2556
2557                 /* Common */
2558                 wqe.num_sge = wr->num_sge;
2559                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2560                         ibdev_err(&rdev->ibdev,
2561                                   "Limit exceeded for Send SGEs");
2562                         rc = -EINVAL;
2563                         goto bad;
2564                 }
2565
2566                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2567                 if (payload_sz < 0) {
2568                         rc = -EINVAL;
2569                         goto bad;
2570                 }
2571                 wqe.wr_id = wr->wr_id;
2572
2573                 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2574
2575                 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2576                 if (!rc)
2577                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2578 bad:
2579                 if (rc) {
2580                         ibdev_err(&rdev->ibdev,
2581                                   "Post send failed opcode = %#x rc = %d",
2582                                   wr->opcode, rc);
2583                         break;
2584                 }
2585                 wr = wr->next;
2586         }
2587         bnxt_qplib_post_send_db(&qp->qplib_qp);
2588         bnxt_ud_qp_hw_stall_workaround(qp);
2589         spin_unlock_irqrestore(&qp->sq_lock, flags);
2590         return rc;
2591 }
2592
2593 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2594                       const struct ib_send_wr **bad_wr)
2595 {
2596         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2597         struct bnxt_qplib_swqe wqe;
2598         int rc = 0, payload_sz = 0;
2599         unsigned long flags;
2600
2601         spin_lock_irqsave(&qp->sq_lock, flags);
2602         while (wr) {
2603                 /* House keeping */
2604                 memset(&wqe, 0, sizeof(wqe));
2605
2606                 /* Common */
2607                 wqe.num_sge = wr->num_sge;
2608                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2609                         ibdev_err(&qp->rdev->ibdev,
2610                                   "Limit exceeded for Send SGEs");
2611                         rc = -EINVAL;
2612                         goto bad;
2613                 }
2614
2615                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2616                 if (payload_sz < 0) {
2617                         rc = -EINVAL;
2618                         goto bad;
2619                 }
2620                 wqe.wr_id = wr->wr_id;
2621
2622                 switch (wr->opcode) {
2623                 case IB_WR_SEND:
2624                 case IB_WR_SEND_WITH_IMM:
2625                         if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2626                                 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2627                                                                payload_sz);
2628                                 if (rc)
2629                                         goto bad;
2630                                 wqe.rawqp1.lflags |=
2631                                         SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2632                         }
2633                         switch (wr->send_flags) {
2634                         case IB_SEND_IP_CSUM:
2635                                 wqe.rawqp1.lflags |=
2636                                         SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2637                                 break;
2638                         default:
2639                                 break;
2640                         }
2641                         fallthrough;
2642                 case IB_WR_SEND_WITH_INV:
2643                         rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2644                         break;
2645                 case IB_WR_RDMA_WRITE:
2646                 case IB_WR_RDMA_WRITE_WITH_IMM:
2647                 case IB_WR_RDMA_READ:
2648                         rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2649                         break;
2650                 case IB_WR_ATOMIC_CMP_AND_SWP:
2651                 case IB_WR_ATOMIC_FETCH_AND_ADD:
2652                         rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2653                         break;
2654                 case IB_WR_RDMA_READ_WITH_INV:
2655                         ibdev_err(&qp->rdev->ibdev,
2656                                   "RDMA Read with Invalidate is not supported");
2657                         rc = -EINVAL;
2658                         goto bad;
2659                 case IB_WR_LOCAL_INV:
2660                         rc = bnxt_re_build_inv_wqe(wr, &wqe);
2661                         break;
2662                 case IB_WR_REG_MR:
2663                         rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2664                         break;
2665                 default:
2666                         /* Unsupported WRs */
2667                         ibdev_err(&qp->rdev->ibdev,
2668                                   "WR (%#x) is not supported", wr->opcode);
2669                         rc = -EINVAL;
2670                         goto bad;
2671                 }
2672                 if (!rc)
2673                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2674 bad:
2675                 if (rc) {
2676                         ibdev_err(&qp->rdev->ibdev,
2677                                   "post_send failed op:%#x qps = %#x rc = %d\n",
2678                                   wr->opcode, qp->qplib_qp.state, rc);
2679                         *bad_wr = wr;
2680                         break;
2681                 }
2682                 wr = wr->next;
2683         }
2684         bnxt_qplib_post_send_db(&qp->qplib_qp);
2685         bnxt_ud_qp_hw_stall_workaround(qp);
2686         spin_unlock_irqrestore(&qp->sq_lock, flags);
2687
2688         return rc;
2689 }
2690
2691 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2692                                        struct bnxt_re_qp *qp,
2693                                        const struct ib_recv_wr *wr)
2694 {
2695         struct bnxt_qplib_swqe wqe;
2696         int rc = 0;
2697
2698         memset(&wqe, 0, sizeof(wqe));
2699         while (wr) {
2700                 /* House keeping */
2701                 memset(&wqe, 0, sizeof(wqe));
2702
2703                 /* Common */
2704                 wqe.num_sge = wr->num_sge;
2705                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2706                         ibdev_err(&rdev->ibdev,
2707                                   "Limit exceeded for Receive SGEs");
2708                         rc = -EINVAL;
2709                         break;
2710                 }
2711                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2712                 wqe.wr_id = wr->wr_id;
2713                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2714
2715                 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2716                 if (rc)
2717                         break;
2718
2719                 wr = wr->next;
2720         }
2721         if (!rc)
2722                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2723         return rc;
2724 }
2725
2726 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
2727                       const struct ib_recv_wr **bad_wr)
2728 {
2729         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2730         struct bnxt_qplib_swqe wqe;
2731         int rc = 0, payload_sz = 0;
2732         unsigned long flags;
2733         u32 count = 0;
2734
2735         spin_lock_irqsave(&qp->rq_lock, flags);
2736         while (wr) {
2737                 /* House keeping */
2738                 memset(&wqe, 0, sizeof(wqe));
2739
2740                 /* Common */
2741                 wqe.num_sge = wr->num_sge;
2742                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2743                         ibdev_err(&qp->rdev->ibdev,
2744                                   "Limit exceeded for Receive SGEs");
2745                         rc = -EINVAL;
2746                         *bad_wr = wr;
2747                         break;
2748                 }
2749
2750                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2751                                                wr->num_sge);
2752                 wqe.wr_id = wr->wr_id;
2753                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2754
2755                 if (ib_qp->qp_type == IB_QPT_GSI &&
2756                     qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
2757                         rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2758                                                               payload_sz);
2759                 if (!rc)
2760                         rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2761                 if (rc) {
2762                         *bad_wr = wr;
2763                         break;
2764                 }
2765
2766                 /* Ring DB if the RQEs posted reaches a threshold value */
2767                 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
2768                         bnxt_qplib_post_recv_db(&qp->qplib_qp);
2769                         count = 0;
2770                 }
2771
2772                 wr = wr->next;
2773         }
2774
2775         if (count)
2776                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2777
2778         spin_unlock_irqrestore(&qp->rq_lock, flags);
2779
2780         return rc;
2781 }
2782
2783 /* Completion Queues */
2784 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
2785 {
2786         struct bnxt_re_cq *cq;
2787         struct bnxt_qplib_nq *nq;
2788         struct bnxt_re_dev *rdev;
2789
2790         cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2791         rdev = cq->rdev;
2792         nq = cq->qplib_cq.nq;
2793
2794         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2795         ib_umem_release(cq->umem);
2796
2797         atomic_dec(&rdev->cq_count);
2798         nq->budget--;
2799         kfree(cq->cql);
2800         return 0;
2801 }
2802
2803 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
2804                       struct ib_udata *udata)
2805 {
2806         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
2807         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2808         struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
2809         int rc, entries;
2810         int cqe = attr->cqe;
2811         struct bnxt_qplib_nq *nq = NULL;
2812         unsigned int nq_alloc_cnt;
2813
2814         if (attr->flags)
2815                 return -EOPNOTSUPP;
2816
2817         /* Validate CQ fields */
2818         if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2819                 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
2820                 return -EINVAL;
2821         }
2822
2823         cq->rdev = rdev;
2824         cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2825
2826         entries = roundup_pow_of_two(cqe + 1);
2827         if (entries > dev_attr->max_cq_wqes + 1)
2828                 entries = dev_attr->max_cq_wqes + 1;
2829
2830         cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
2831         cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
2832         if (udata) {
2833                 struct bnxt_re_cq_req req;
2834                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
2835                         udata, struct bnxt_re_ucontext, ib_uctx);
2836                 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2837                         rc = -EFAULT;
2838                         goto fail;
2839                 }
2840
2841                 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
2842                                        entries * sizeof(struct cq_base),
2843                                        IB_ACCESS_LOCAL_WRITE);
2844                 if (IS_ERR(cq->umem)) {
2845                         rc = PTR_ERR(cq->umem);
2846                         goto fail;
2847                 }
2848                 cq->qplib_cq.sg_info.umem = cq->umem;
2849                 cq->qplib_cq.dpi = &uctx->dpi;
2850         } else {
2851                 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2852                 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2853                                   GFP_KERNEL);
2854                 if (!cq->cql) {
2855                         rc = -ENOMEM;
2856                         goto fail;
2857                 }
2858
2859                 cq->qplib_cq.dpi = &rdev->dpi_privileged;
2860         }
2861         /*
2862          * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
2863          * used for getting the NQ index.
2864          */
2865         nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
2866         nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
2867         cq->qplib_cq.max_wqe = entries;
2868         cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
2869         cq->qplib_cq.nq = nq;
2870
2871         rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2872         if (rc) {
2873                 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
2874                 goto fail;
2875         }
2876
2877         cq->ib_cq.cqe = entries;
2878         cq->cq_period = cq->qplib_cq.period;
2879         nq->budget++;
2880
2881         atomic_inc(&rdev->cq_count);
2882         spin_lock_init(&cq->cq_lock);
2883
2884         if (udata) {
2885                 struct bnxt_re_cq_resp resp;
2886
2887                 resp.cqid = cq->qplib_cq.id;
2888                 resp.tail = cq->qplib_cq.hwq.cons;
2889                 resp.phase = cq->qplib_cq.period;
2890                 resp.rsvd = 0;
2891                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2892                 if (rc) {
2893                         ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
2894                         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2895                         goto c2fail;
2896                 }
2897         }
2898
2899         return 0;
2900
2901 c2fail:
2902         ib_umem_release(cq->umem);
2903 fail:
2904         kfree(cq->cql);
2905         return rc;
2906 }
2907
2908 static u8 __req_to_ib_wc_status(u8 qstatus)
2909 {
2910         switch (qstatus) {
2911         case CQ_REQ_STATUS_OK:
2912                 return IB_WC_SUCCESS;
2913         case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2914                 return IB_WC_BAD_RESP_ERR;
2915         case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2916                 return IB_WC_LOC_LEN_ERR;
2917         case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2918                 return IB_WC_LOC_QP_OP_ERR;
2919         case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2920                 return IB_WC_LOC_PROT_ERR;
2921         case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2922                 return IB_WC_GENERAL_ERR;
2923         case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2924                 return IB_WC_REM_INV_REQ_ERR;
2925         case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2926                 return IB_WC_REM_ACCESS_ERR;
2927         case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2928                 return IB_WC_REM_OP_ERR;
2929         case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2930                 return IB_WC_RNR_RETRY_EXC_ERR;
2931         case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2932                 return IB_WC_RETRY_EXC_ERR;
2933         case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2934                 return IB_WC_WR_FLUSH_ERR;
2935         default:
2936                 return IB_WC_GENERAL_ERR;
2937         }
2938         return 0;
2939 }
2940
2941 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2942 {
2943         switch (qstatus) {
2944         case CQ_RES_RAWETH_QP1_STATUS_OK:
2945                 return IB_WC_SUCCESS;
2946         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2947                 return IB_WC_LOC_ACCESS_ERR;
2948         case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2949                 return IB_WC_LOC_LEN_ERR;
2950         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2951                 return IB_WC_LOC_PROT_ERR;
2952         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2953                 return IB_WC_LOC_QP_OP_ERR;
2954         case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2955                 return IB_WC_GENERAL_ERR;
2956         case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2957                 return IB_WC_WR_FLUSH_ERR;
2958         case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2959                 return IB_WC_WR_FLUSH_ERR;
2960         default:
2961                 return IB_WC_GENERAL_ERR;
2962         }
2963 }
2964
2965 static u8 __rc_to_ib_wc_status(u8 qstatus)
2966 {
2967         switch (qstatus) {
2968         case CQ_RES_RC_STATUS_OK:
2969                 return IB_WC_SUCCESS;
2970         case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2971                 return IB_WC_LOC_ACCESS_ERR;
2972         case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2973                 return IB_WC_LOC_LEN_ERR;
2974         case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2975                 return IB_WC_LOC_PROT_ERR;
2976         case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2977                 return IB_WC_LOC_QP_OP_ERR;
2978         case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2979                 return IB_WC_GENERAL_ERR;
2980         case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2981                 return IB_WC_REM_INV_REQ_ERR;
2982         case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2983                 return IB_WC_WR_FLUSH_ERR;
2984         case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2985                 return IB_WC_WR_FLUSH_ERR;
2986         default:
2987                 return IB_WC_GENERAL_ERR;
2988         }
2989 }
2990
2991 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2992 {
2993         switch (cqe->type) {
2994         case BNXT_QPLIB_SWQE_TYPE_SEND:
2995                 wc->opcode = IB_WC_SEND;
2996                 break;
2997         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2998                 wc->opcode = IB_WC_SEND;
2999                 wc->wc_flags |= IB_WC_WITH_IMM;
3000                 break;
3001         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3002                 wc->opcode = IB_WC_SEND;
3003                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3004                 break;
3005         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3006                 wc->opcode = IB_WC_RDMA_WRITE;
3007                 break;
3008         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3009                 wc->opcode = IB_WC_RDMA_WRITE;
3010                 wc->wc_flags |= IB_WC_WITH_IMM;
3011                 break;
3012         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3013                 wc->opcode = IB_WC_RDMA_READ;
3014                 break;
3015         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3016                 wc->opcode = IB_WC_COMP_SWAP;
3017                 break;
3018         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3019                 wc->opcode = IB_WC_FETCH_ADD;
3020                 break;
3021         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3022                 wc->opcode = IB_WC_LOCAL_INV;
3023                 break;
3024         case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3025                 wc->opcode = IB_WC_REG_MR;
3026                 break;
3027         default:
3028                 wc->opcode = IB_WC_SEND;
3029                 break;
3030         }
3031
3032         wc->status = __req_to_ib_wc_status(cqe->status);
3033 }
3034
3035 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3036                                      u16 raweth_qp1_flags2)
3037 {
3038         bool is_ipv6 = false, is_ipv4 = false;
3039
3040         /* raweth_qp1_flags Bit 9-6 indicates itype */
3041         if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3042             != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3043                 return -1;
3044
3045         if (raweth_qp1_flags2 &
3046             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3047             raweth_qp1_flags2 &
3048             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3049                 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3050                 (raweth_qp1_flags2 &
3051                  CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3052                         (is_ipv6 = true) : (is_ipv4 = true);
3053                 return ((is_ipv6) ?
3054                          BNXT_RE_ROCEV2_IPV6_PACKET :
3055                          BNXT_RE_ROCEV2_IPV4_PACKET);
3056         } else {
3057                 return BNXT_RE_ROCE_V1_PACKET;
3058         }
3059 }
3060
3061 static int bnxt_re_to_ib_nw_type(int nw_type)
3062 {
3063         u8 nw_hdr_type = 0xFF;
3064
3065         switch (nw_type) {
3066         case BNXT_RE_ROCE_V1_PACKET:
3067                 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3068                 break;
3069         case BNXT_RE_ROCEV2_IPV4_PACKET:
3070                 nw_hdr_type = RDMA_NETWORK_IPV4;
3071                 break;
3072         case BNXT_RE_ROCEV2_IPV6_PACKET:
3073                 nw_hdr_type = RDMA_NETWORK_IPV6;
3074                 break;
3075         }
3076         return nw_hdr_type;
3077 }
3078
3079 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3080                                        void *rq_hdr_buf)
3081 {
3082         u8 *tmp_buf = NULL;
3083         struct ethhdr *eth_hdr;
3084         u16 eth_type;
3085         bool rc = false;
3086
3087         tmp_buf = (u8 *)rq_hdr_buf;
3088         /*
3089          * If dest mac is not same as I/F mac, this could be a
3090          * loopback address or multicast address, check whether
3091          * it is a loopback packet
3092          */
3093         if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3094                 tmp_buf += 4;
3095                 /* Check the  ether type */
3096                 eth_hdr = (struct ethhdr *)tmp_buf;
3097                 eth_type = ntohs(eth_hdr->h_proto);
3098                 switch (eth_type) {
3099                 case ETH_P_IBOE:
3100                         rc = true;
3101                         break;
3102                 case ETH_P_IP:
3103                 case ETH_P_IPV6: {
3104                         u32 len;
3105                         struct udphdr *udp_hdr;
3106
3107                         len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3108                                                       sizeof(struct ipv6hdr));
3109                         tmp_buf += sizeof(struct ethhdr) + len;
3110                         udp_hdr = (struct udphdr *)tmp_buf;
3111                         if (ntohs(udp_hdr->dest) ==
3112                                     ROCE_V2_UDP_DPORT)
3113                                 rc = true;
3114                         break;
3115                         }
3116                 default:
3117                         break;
3118                 }
3119         }
3120
3121         return rc;
3122 }
3123
3124 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3125                                          struct bnxt_qplib_cqe *cqe)
3126 {
3127         struct bnxt_re_dev *rdev = gsi_qp->rdev;
3128         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3129         struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3130         struct bnxt_re_ah *gsi_sah;
3131         struct ib_send_wr *swr;
3132         struct ib_ud_wr udwr;
3133         struct ib_recv_wr rwr;
3134         int pkt_type = 0;
3135         u32 tbl_idx;
3136         void *rq_hdr_buf;
3137         dma_addr_t rq_hdr_buf_map;
3138         dma_addr_t shrq_hdr_buf_map;
3139         u32 offset = 0;
3140         u32 skip_bytes = 0;
3141         struct ib_sge s_sge[2];
3142         struct ib_sge r_sge[2];
3143         int rc;
3144
3145         memset(&udwr, 0, sizeof(udwr));
3146         memset(&rwr, 0, sizeof(rwr));
3147         memset(&s_sge, 0, sizeof(s_sge));
3148         memset(&r_sge, 0, sizeof(r_sge));
3149
3150         swr = &udwr.wr;
3151         tbl_idx = cqe->wr_id;
3152
3153         rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3154                         (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3155         rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3156                                                           tbl_idx);
3157
3158         /* Shadow QP header buffer */
3159         shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3160                                                             tbl_idx);
3161         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3162
3163         /* Store this cqe */
3164         memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3165         sqp_entry->qp1_qp = gsi_qp;
3166
3167         /* Find packet type from the cqe */
3168
3169         pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3170                                              cqe->raweth_qp1_flags2);
3171         if (pkt_type < 0) {
3172                 ibdev_err(&rdev->ibdev, "Invalid packet\n");
3173                 return -EINVAL;
3174         }
3175
3176         /* Adjust the offset for the user buffer and post in the rq */
3177
3178         if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3179                 offset = 20;
3180
3181         /*
3182          * QP1 loopback packet has 4 bytes of internal header before
3183          * ether header. Skip these four bytes.
3184          */
3185         if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3186                 skip_bytes = 4;
3187
3188         /* First send SGE . Skip the ether header*/
3189         s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3190                         + skip_bytes;
3191         s_sge[0].lkey = 0xFFFFFFFF;
3192         s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3193                                 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3194
3195         /* Second Send SGE */
3196         s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3197                         BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3198         if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3199                 s_sge[1].addr += 8;
3200         s_sge[1].lkey = 0xFFFFFFFF;
3201         s_sge[1].length = 256;
3202
3203         /* First recv SGE */
3204
3205         r_sge[0].addr = shrq_hdr_buf_map;
3206         r_sge[0].lkey = 0xFFFFFFFF;
3207         r_sge[0].length = 40;
3208
3209         r_sge[1].addr = sqp_entry->sge.addr + offset;
3210         r_sge[1].lkey = sqp_entry->sge.lkey;
3211         r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3212
3213         /* Create receive work request */
3214         rwr.num_sge = 2;
3215         rwr.sg_list = r_sge;
3216         rwr.wr_id = tbl_idx;
3217         rwr.next = NULL;
3218
3219         rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
3220         if (rc) {
3221                 ibdev_err(&rdev->ibdev,
3222                           "Failed to post Rx buffers to shadow QP");
3223                 return -ENOMEM;
3224         }
3225
3226         swr->num_sge = 2;
3227         swr->sg_list = s_sge;
3228         swr->wr_id = tbl_idx;
3229         swr->opcode = IB_WR_SEND;
3230         swr->next = NULL;
3231         gsi_sah = rdev->gsi_ctx.gsi_sah;
3232         udwr.ah = &gsi_sah->ib_ah;
3233         udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3234         udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3235
3236         /* post data received  in the send queue */
3237         rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
3238
3239         return 0;
3240 }
3241
3242 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3243                                           struct bnxt_qplib_cqe *cqe)
3244 {
3245         wc->opcode = IB_WC_RECV;
3246         wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3247         wc->wc_flags |= IB_WC_GRH;
3248 }
3249
3250 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3251                                         u16 vlan_id)
3252 {
3253         /*
3254          * Check if the vlan is configured in the host.  If not configured, it
3255          * can be a transparent VLAN. So dont report the vlan id.
3256          */
3257         if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3258                                       htons(ETH_P_8021Q), vlan_id))
3259                 return false;
3260         return true;
3261 }
3262
3263 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3264                                 u16 *vid, u8 *sl)
3265 {
3266         bool ret = false;
3267         u32 metadata;
3268         u16 tpid;
3269
3270         metadata = orig_cqe->raweth_qp1_metadata;
3271         if (orig_cqe->raweth_qp1_flags2 &
3272                 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3273                 tpid = ((metadata &
3274                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3275                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3276                 if (tpid == ETH_P_8021Q) {
3277                         *vid = metadata &
3278                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3279                         *sl = (metadata &
3280                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3281                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3282                         ret = true;
3283                 }
3284         }
3285
3286         return ret;
3287 }
3288
3289 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3290                                       struct bnxt_qplib_cqe *cqe)
3291 {
3292         wc->opcode = IB_WC_RECV;
3293         wc->status = __rc_to_ib_wc_status(cqe->status);
3294
3295         if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3296                 wc->wc_flags |= IB_WC_WITH_IMM;
3297         if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3298                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3299         if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3300             (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3301                 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3302 }
3303
3304 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3305                                              struct ib_wc *wc,
3306                                              struct bnxt_qplib_cqe *cqe)
3307 {
3308         struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3309         struct bnxt_re_qp *gsi_qp = NULL;
3310         struct bnxt_qplib_cqe *orig_cqe = NULL;
3311         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3312         int nw_type;
3313         u32 tbl_idx;
3314         u16 vlan_id;
3315         u8 sl;
3316
3317         tbl_idx = cqe->wr_id;
3318
3319         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3320         gsi_qp = sqp_entry->qp1_qp;
3321         orig_cqe = &sqp_entry->cqe;
3322
3323         wc->wr_id = sqp_entry->wrid;
3324         wc->byte_len = orig_cqe->length;
3325         wc->qp = &gsi_qp->ib_qp;
3326
3327         wc->ex.imm_data = orig_cqe->immdata;
3328         wc->src_qp = orig_cqe->src_qp;
3329         memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
3330         if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
3331                 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3332                         wc->vlan_id = vlan_id;
3333                         wc->sl = sl;
3334                         wc->wc_flags |= IB_WC_WITH_VLAN;
3335                 }
3336         }
3337         wc->port_num = 1;
3338         wc->vendor_err = orig_cqe->status;
3339
3340         wc->opcode = IB_WC_RECV;
3341         wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
3342         wc->wc_flags |= IB_WC_GRH;
3343
3344         nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
3345                                             orig_cqe->raweth_qp1_flags2);
3346         if (nw_type >= 0) {
3347                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3348                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3349         }
3350 }
3351
3352 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3353                                       struct ib_wc *wc,
3354                                       struct bnxt_qplib_cqe *cqe)
3355 {
3356         u8 nw_type;
3357
3358         wc->opcode = IB_WC_RECV;
3359         wc->status = __rc_to_ib_wc_status(cqe->status);
3360
3361         if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
3362                 wc->wc_flags |= IB_WC_WITH_IMM;
3363         /* report only on GSI QP for Thor */
3364         if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3365                 wc->wc_flags |= IB_WC_GRH;
3366                 memcpy(wc->smac, cqe->smac, ETH_ALEN);
3367                 wc->wc_flags |= IB_WC_WITH_SMAC;
3368                 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
3369                         wc->vlan_id = (cqe->cfa_meta & 0xFFF);
3370                         if (wc->vlan_id < 0x1000)
3371                                 wc->wc_flags |= IB_WC_WITH_VLAN;
3372                 }
3373                 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3374                            CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3375                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3376                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3377         }
3378
3379 }
3380
3381 static int send_phantom_wqe(struct bnxt_re_qp *qp)
3382 {
3383         struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3384         unsigned long flags;
3385         int rc = 0;
3386
3387         spin_lock_irqsave(&qp->sq_lock, flags);
3388
3389         rc = bnxt_re_bind_fence_mw(lib_qp);
3390         if (!rc) {
3391                 lib_qp->sq.phantom_wqe_cnt++;
3392                 ibdev_dbg(&qp->rdev->ibdev,
3393                           "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3394                           lib_qp->id, lib_qp->sq.hwq.prod,
3395                           HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3396                           lib_qp->sq.phantom_wqe_cnt);
3397         }
3398
3399         spin_unlock_irqrestore(&qp->sq_lock, flags);
3400         return rc;
3401 }
3402
3403 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3404 {
3405         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3406         struct bnxt_re_qp *qp, *sh_qp;
3407         struct bnxt_qplib_cqe *cqe;
3408         int i, ncqe, budget;
3409         struct bnxt_qplib_q *sq;
3410         struct bnxt_qplib_qp *lib_qp;
3411         u32 tbl_idx;
3412         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3413         unsigned long flags;
3414
3415         spin_lock_irqsave(&cq->cq_lock, flags);
3416         budget = min_t(u32, num_entries, cq->max_cql);
3417         num_entries = budget;
3418         if (!cq->cql) {
3419                 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
3420                 goto exit;
3421         }
3422         cqe = &cq->cql[0];
3423         while (budget) {
3424                 lib_qp = NULL;
3425                 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
3426                 if (lib_qp) {
3427                         sq = &lib_qp->sq;
3428                         if (sq->send_phantom) {
3429                                 qp = container_of(lib_qp,
3430                                                   struct bnxt_re_qp, qplib_qp);
3431                                 if (send_phantom_wqe(qp) == -ENOMEM)
3432                                         ibdev_err(&cq->rdev->ibdev,
3433                                                   "Phantom failed! Scheduled to send again\n");
3434                                 else
3435                                         sq->send_phantom = false;
3436                         }
3437                 }
3438                 if (ncqe < budget)
3439                         ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
3440                                                               cqe + ncqe,
3441                                                               budget - ncqe);
3442
3443                 if (!ncqe)
3444                         break;
3445
3446                 for (i = 0; i < ncqe; i++, cqe++) {
3447                         /* Transcribe each qplib_wqe back to ib_wc */
3448                         memset(wc, 0, sizeof(*wc));
3449
3450                         wc->wr_id = cqe->wr_id;
3451                         wc->byte_len = cqe->length;
3452                         qp = container_of
3453                                 ((struct bnxt_qplib_qp *)
3454                                  (unsigned long)(cqe->qp_handle),
3455                                  struct bnxt_re_qp, qplib_qp);
3456                         wc->qp = &qp->ib_qp;
3457                         wc->ex.imm_data = cqe->immdata;
3458                         wc->src_qp = cqe->src_qp;
3459                         memcpy(wc->smac, cqe->smac, ETH_ALEN);
3460                         wc->port_num = 1;
3461                         wc->vendor_err = cqe->status;
3462
3463                         switch (cqe->opcode) {
3464                         case CQ_BASE_CQE_TYPE_REQ:
3465                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3466                                 if (sh_qp &&
3467                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3468                                         /* Handle this completion with
3469                                          * the stored completion
3470                                          */
3471                                         memset(wc, 0, sizeof(*wc));
3472                                         continue;
3473                                 }
3474                                 bnxt_re_process_req_wc(wc, cqe);
3475                                 break;
3476                         case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3477                                 if (!cqe->status) {
3478                                         int rc = 0;
3479
3480                                         rc = bnxt_re_process_raw_qp_pkt_rx
3481                              &n