drivers/infiniband/hw/hns/hns_roce_hw_v2.c

   1 /*
   2  * Copyright (c) 2016-2017 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/acpi.h>
  34 #include <linux/etherdevice.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/kernel.h>
  37 #include <linux/types.h>
  38 #include <net/addrconf.h>
  39 #include <rdma/ib_addr.h>
  40 #include <rdma/ib_cache.h>
  41 #include <rdma/ib_umem.h>
  42 #include <rdma/uverbs_ioctl.h>
  43
  44 #include "hnae3.h"
  45 #include "hns_roce_common.h"
  46 #include "hns_roce_device.h"
  47 #include "hns_roce_cmd.h"
  48 #include "hns_roce_hem.h"
  49 #include "hns_roce_hw_v2.h"
  50
  51 enum {
  52         CMD_RST_PRC_OTHERS,
  53         CMD_RST_PRC_SUCCESS,
  54         CMD_RST_PRC_EBUSY,
  55 };
  56
  57 static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
  58                                    struct ib_sge *sg)
  59 {
  60         dseg->lkey = cpu_to_le32(sg->lkey);
  61         dseg->addr = cpu_to_le64(sg->addr);
  62         dseg->len  = cpu_to_le32(sg->length);
  63 }
  64
  65 /*
  66  * mapped-value = 1 + real-value
  67  * The hns wr opcode real value is start from 0, In order to distinguish between
  68  * initialized and uninitialized map values, we plus 1 to the actual value when
  69  * defining the mapping, so that the validity can be identified by checking the
  70  * mapped value is greater than 0.
  71  */
  72 #define HR_OPC_MAP(ib_key, hr_key) \
  73                 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key
  74
  75 static const u32 hns_roce_op_code[] = {
  76         HR_OPC_MAP(RDMA_WRITE,                  RDMA_WRITE),
  77         HR_OPC_MAP(RDMA_WRITE_WITH_IMM,         RDMA_WRITE_WITH_IMM),
  78         HR_OPC_MAP(SEND,                        SEND),
  79         HR_OPC_MAP(SEND_WITH_IMM,               SEND_WITH_IMM),
  80         HR_OPC_MAP(RDMA_READ,                   RDMA_READ),
  81         HR_OPC_MAP(ATOMIC_CMP_AND_SWP,          ATOM_CMP_AND_SWAP),
  82         HR_OPC_MAP(ATOMIC_FETCH_AND_ADD,        ATOM_FETCH_AND_ADD),
  83         HR_OPC_MAP(SEND_WITH_INV,               SEND_WITH_INV),
  84         HR_OPC_MAP(LOCAL_INV,                   LOCAL_INV),
  85         HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP,   ATOM_MSK_CMP_AND_SWAP),
  86         HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),
  87         HR_OPC_MAP(REG_MR,                      FAST_REG_PMR),
  88 };
  89
  90 static u32 to_hr_opcode(u32 ib_opcode)
  91 {
  92         if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
  93                 return HNS_ROCE_V2_WQE_OP_MASK;
  94
  95         return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
  96                                              HNS_ROCE_V2_WQE_OP_MASK;
  97 }
  98
  99 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 100                          const struct ib_reg_wr *wr)
 101 {
 102         struct hns_roce_wqe_frmr_seg *fseg =
 103                 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 104         struct hns_roce_mr *mr = to_hr_mr(wr->mr);
 105         u64 pbl_ba;
 106
 107         /* use ib_access_flags */
 108         hr_reg_write_bool(fseg, FRMR_BIND_EN, wr->access & IB_ACCESS_MW_BIND);
 109         hr_reg_write_bool(fseg, FRMR_ATOMIC,
 110                           wr->access & IB_ACCESS_REMOTE_ATOMIC);
 111         hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ);
 112         hr_reg_write_bool(fseg, FRMR_RW, wr->access & IB_ACCESS_REMOTE_WRITE);
 113         hr_reg_write_bool(fseg, FRMR_LW, wr->access & IB_ACCESS_LOCAL_WRITE);
 114
 115         /* Data structure reuse may lead to confusion */
 116         pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
 117         rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
 118         rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
 119
 120         rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
 121         rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
 122         rc_sq_wqe->rkey = cpu_to_le32(wr->key);
 123         rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
 124
 125         hr_reg_write(fseg, FRMR_PBL_SIZE, mr->npages);
 126         hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ,
 127                      to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
 128         hr_reg_clear(fseg, FRMR_BLK_MODE);
 129 }
 130
 131 static void set_atomic_seg(const struct ib_send_wr *wr,
 132                            struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 133                            unsigned int valid_num_sge)
 134 {
 135         struct hns_roce_v2_wqe_data_seg *dseg =
 136                 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 137         struct hns_roce_wqe_atomic_seg *aseg =
 138                 (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
 139
 140         set_data_seg_v2(dseg, wr->sg_list);
 141
 142         if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 143                 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
 144                 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
 145         } else {
 146                 aseg->fetchadd_swap_data =
 147                         cpu_to_le64(atomic_wr(wr)->compare_add);
 148                 aseg->cmp_data = 0;
 149         }
 150
 151         roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 152                        V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
 153 }
 154
 155 static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,
 156                                  const struct ib_send_wr *wr,
 157                                  unsigned int *sge_idx, u32 msg_len)
 158 {
 159         struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
 160         unsigned int dseg_len = sizeof(struct hns_roce_v2_wqe_data_seg);
 161         unsigned int ext_sge_sz = qp->sq.max_gs * dseg_len;
 162         unsigned int left_len_in_pg;
 163         unsigned int idx = *sge_idx;
 164         unsigned int i = 0;
 165         unsigned int len;
 166         void *addr;
 167         void *dseg;
 168
 169         if (msg_len > ext_sge_sz) {
 170                 ibdev_err(ibdev,
 171                           "no enough extended sge space for inline data.\n");
 172                 return -EINVAL;
 173         }
 174
 175         dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
 176         left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;
 177         len = wr->sg_list[0].length;
 178         addr = (void *)(unsigned long)(wr->sg_list[0].addr);
 179
 180         /* When copying data to extended sge space, the left length in page may
 181          * not long enough for current user's sge. So the data should be
 182          * splited into several parts, one in the first page, and the others in
 183          * the subsequent pages.
 184          */
 185         while (1) {
 186                 if (len <= left_len_in_pg) {
 187                         memcpy(dseg, addr, len);
 188
 189                         idx += len / dseg_len;
 190
 191                         i++;
 192                         if (i >= wr->num_sge)
 193                                 break;
 194
 195                         left_len_in_pg -= len;
 196                         len = wr->sg_list[i].length;
 197                         addr = (void *)(unsigned long)(wr->sg_list[i].addr);
 198                         dseg += len;
 199                 } else {
 200                         memcpy(dseg, addr, left_len_in_pg);
 201
 202                         len -= left_len_in_pg;
 203                         addr += left_len_in_pg;
 204                         idx += left_len_in_pg / dseg_len;
 205                         dseg = hns_roce_get_extend_sge(qp,
 206                                                 idx & (qp->sge.sge_cnt - 1));
 207                         left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;
 208                 }
 209         }
 210
 211         *sge_idx = idx;
 212
 213         return 0;
 214 }
 215
 216 static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,
 217                            unsigned int *sge_ind, unsigned int cnt)
 218 {
 219         struct hns_roce_v2_wqe_data_seg *dseg;
 220         unsigned int idx = *sge_ind;
 221
 222         while (cnt > 0) {
 223                 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
 224                 if (likely(sge->length)) {
 225                         set_data_seg_v2(dseg, sge);
 226                         idx++;
 227                         cnt--;
 228                 }
 229                 sge++;
 230         }
 231
 232         *sge_ind = idx;
 233 }
 234
 235 static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)
 236 {
 237         struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 238         int mtu = ib_mtu_enum_to_int(qp->path_mtu);
 239
 240         if (len > qp->max_inline_data || len > mtu) {
 241                 ibdev_err(&hr_dev->ib_dev,
 242                           "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",
 243                           len, qp->max_inline_data, mtu);
 244                 return false;
 245         }
 246
 247         return true;
 248 }
 249
 250 static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
 251                       struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 252                       unsigned int *sge_idx)
 253 {
 254         struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 255         u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);
 256         struct ib_device *ibdev = &hr_dev->ib_dev;
 257         unsigned int curr_idx = *sge_idx;
 258         void *dseg = rc_sq_wqe;
 259         unsigned int i;
 260         int ret;
 261
 262         if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
 263                 ibdev_err(ibdev, "invalid inline parameters!\n");
 264                 return -EINVAL;
 265         }
 266
 267         if (!check_inl_data_len(qp, msg_len))
 268                 return -EINVAL;
 269
 270         dseg += sizeof(struct hns_roce_v2_rc_send_wqe);
 271
 272         if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {
 273                 roce_set_bit(rc_sq_wqe->byte_20,
 274                              V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 0);
 275
 276                 for (i = 0; i < wr->num_sge; i++) {
 277                         memcpy(dseg, ((void *)wr->sg_list[i].addr),
 278                                wr->sg_list[i].length);
 279                         dseg += wr->sg_list[i].length;
 280                 }
 281         } else {
 282                 roce_set_bit(rc_sq_wqe->byte_20,
 283                              V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 1);
 284
 285                 ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);
 286                 if (ret)
 287                         return ret;
 288
 289                 roce_set_field(rc_sq_wqe->byte_16,
 290                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 291                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
 292                                curr_idx - *sge_idx);
 293         }
 294
 295         *sge_idx = curr_idx;
 296
 297         return 0;
 298 }
 299
 300 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 301                              struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 302                              unsigned int *sge_ind,
 303                              unsigned int valid_num_sge)
 304 {
 305         struct hns_roce_v2_wqe_data_seg *dseg =
 306                 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 307         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 308         int j = 0;
 309         int i;
 310
 311         roce_set_field(rc_sq_wqe->byte_20,
 312                        V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 313                        V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 314                        (*sge_ind) & (qp->sge.sge_cnt - 1));
 315
 316         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
 317                      !!(wr->send_flags & IB_SEND_INLINE));
 318         if (wr->send_flags & IB_SEND_INLINE)
 319                 return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);
 320
 321         if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
 322                 for (i = 0; i < wr->num_sge; i++) {
 323                         if (likely(wr->sg_list[i].length)) {
 324                                 set_data_seg_v2(dseg, wr->sg_list + i);
 325                                 dseg++;
 326                         }
 327                 }
 328         } else {
 329                 for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {
 330                         if (likely(wr->sg_list[i].length)) {
 331                                 set_data_seg_v2(dseg, wr->sg_list + i);
 332                                 dseg++;
 333                                 j++;
 334                         }
 335                 }
 336
 337                 set_extend_sge(qp, wr->sg_list + i, sge_ind,
 338                                valid_num_sge - HNS_ROCE_SGE_IN_WQE);
 339         }
 340
 341         roce_set_field(rc_sq_wqe->byte_16,
 342                        V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 343                        V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
 344
 345         return 0;
 346 }
 347
 348 static int check_send_valid(struct hns_roce_dev *hr_dev,
 349                             struct hns_roce_qp *hr_qp)
 350 {
 351         struct ib_device *ibdev = &hr_dev->ib_dev;
 352         struct ib_qp *ibqp = &hr_qp->ibqp;
 353
 354         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
 355                      ibqp->qp_type != IB_QPT_GSI &&
 356                      ibqp->qp_type != IB_QPT_UD)) {
 357                 ibdev_err(ibdev, "Not supported QP(0x%x)type!\n",
 358                           ibqp->qp_type);
 359                 return -EOPNOTSUPP;
 360         } else if (unlikely(hr_qp->state == IB_QPS_RESET ||
 361                    hr_qp->state == IB_QPS_INIT ||
 362                    hr_qp->state == IB_QPS_RTR)) {
 363                 ibdev_err(ibdev, "failed to post WQE, QP state %u!\n",
 364                           hr_qp->state);
 365                 return -EINVAL;
 366         } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
 367                 ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",
 368                           hr_dev->state);
 369                 return -EIO;
 370         }
 371
 372         return 0;
 373 }
 374
 375 static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
 376                                     unsigned int *sge_len)
 377 {
 378         unsigned int valid_num = 0;
 379         unsigned int len = 0;
 380         int i;
 381
 382         for (i = 0; i < wr->num_sge; i++) {
 383                 if (likely(wr->sg_list[i].length)) {
 384                         len += wr->sg_list[i].length;
 385                         valid_num++;
 386                 }
 387         }
 388
 389         *sge_len = len;
 390         return valid_num;
 391 }
 392
 393 static __le32 get_immtdata(const struct ib_send_wr *wr)
 394 {
 395         switch (wr->opcode) {
 396         case IB_WR_SEND_WITH_IMM:
 397         case IB_WR_RDMA_WRITE_WITH_IMM:
 398                 return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 399         default:
 400                 return 0;
 401         }
 402 }
 403
 404 static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 405                          const struct ib_send_wr *wr)
 406 {
 407         u32 ib_op = wr->opcode;
 408
 409         if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)
 410                 return -EINVAL;
 411
 412         ud_sq_wqe->immtdata = get_immtdata(wr);
 413
 414         roce_set_field(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
 415                        V2_UD_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
 416
 417         return 0;
 418 }
 419
 420 static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 421                       struct hns_roce_ah *ah)
 422 {
 423         struct ib_device *ib_dev = ah->ibah.device;
 424         struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
 425
 426         roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
 427                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
 428
 429         roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
 430                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
 431         roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
 432                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
 433         roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
 434                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
 435
 436         if (WARN_ON(ah->av.sl > MAX_SERVICE_LEVEL))
 437                 return -EINVAL;
 438
 439         roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
 440                        V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
 441
 442         ud_sq_wqe->sgid_index = ah->av.gid_index;
 443
 444         memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);
 445         memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);
 446
 447         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 448                 return 0;
 449
 450         roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
 451                      ah->av.vlan_en);
 452         roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
 453                        V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
 454
 455         return 0;
 456 }
 457
 458 static inline int set_ud_wqe(struct hns_roce_qp *qp,
 459                              const struct ib_send_wr *wr,
 460                              void *wqe, unsigned int *sge_idx,
 461                              unsigned int owner_bit)
 462 {
 463         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 464         struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
 465         unsigned int curr_idx = *sge_idx;
 466         unsigned int valid_num_sge;
 467         u32 msg_len = 0;
 468         int ret;
 469
 470         valid_num_sge = calc_wr_sge_num(wr, &msg_len);
 471
 472         ret = set_ud_opcode(ud_sq_wqe, wr);
 473         if (WARN_ON(ret))
 474                 return ret;
 475
 476         ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
 477
 478         roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
 479                      !!(wr->send_flags & IB_SEND_SIGNALED));
 480
 481         roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
 482                      !!(wr->send_flags & IB_SEND_SOLICITED));
 483
 484         roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
 485                        V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);
 486
 487         roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
 488                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
 489
 490         roce_set_field(ud_sq_wqe->byte_20,
 491                        V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 492                        V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 493                        curr_idx & (qp->sge.sge_cnt - 1));
 494
 495         ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
 496                           qp->qkey : ud_wr(wr)->remote_qkey);
 497         roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
 498                        V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
 499
 500         ret = fill_ud_av(ud_sq_wqe, ah);
 501         if (ret)
 502                 return ret;
 503
 504         qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
 505
 506         set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
 507
 508         /*
 509          * The pipeline can sequentially post all valid WQEs into WQ buffer,
 510          * including new WQEs waiting for the doorbell to update the PI again.
 511          * Therefore, the owner bit of WQE MUST be updated after all fields
 512          * and extSGEs have been written into DDR instead of cache.
 513          */
 514         if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
 515                 dma_wmb();
 516
 517         *sge_idx = curr_idx;
 518         roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
 519                      owner_bit);
 520
 521         return 0;
 522 }
 523
 524 static int set_rc_opcode(struct hns_roce_dev *hr_dev,
 525                          struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 526                          const struct ib_send_wr *wr)
 527 {
 528         u32 ib_op = wr->opcode;
 529         int ret = 0;
 530
 531         rc_sq_wqe->immtdata = get_immtdata(wr);
 532
 533         switch (ib_op) {
 534         case IB_WR_RDMA_READ:
 535         case IB_WR_RDMA_WRITE:
 536         case IB_WR_RDMA_WRITE_WITH_IMM:
 537                 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
 538                 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
 539                 break;
 540         case IB_WR_SEND:
 541         case IB_WR_SEND_WITH_IMM:
 542                 break;
 543         case IB_WR_ATOMIC_CMP_AND_SWP:
 544         case IB_WR_ATOMIC_FETCH_AND_ADD:
 545                 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
 546                 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
 547                 break;
 548         case IB_WR_REG_MR:
 549                 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 550                         set_frmr_seg(rc_sq_wqe, reg_wr(wr));
 551                 else
 552                         ret = -EOPNOTSUPP;
 553                 break;
 554         case IB_WR_LOCAL_INV:
 555                 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
 556                 fallthrough;
 557         case IB_WR_SEND_WITH_INV:
 558                 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
 559                 break;
 560         default:
 561                 ret = -EINVAL;
 562         }
 563
 564         if (unlikely(ret))
 565                 return ret;
 566
 567         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 568                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
 569
 570         return ret;
 571 }
 572 static inline int set_rc_wqe(struct hns_roce_qp *qp,
 573                              const struct ib_send_wr *wr,
 574                              void *wqe, unsigned int *sge_idx,
 575                              unsigned int owner_bit)
 576 {
 577         struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 578         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 579         unsigned int curr_idx = *sge_idx;
 580         unsigned int valid_num_sge;
 581         u32 msg_len = 0;
 582         int ret;
 583
 584         valid_num_sge = calc_wr_sge_num(wr, &msg_len);
 585
 586         rc_sq_wqe->msg_len = cpu_to_le32(msg_len);
 587
 588         ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr);
 589         if (WARN_ON(ret))
 590                 return ret;
 591
 592         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S,
 593                      (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
 594
 595         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S,
 596                      (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 597
 598         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
 599                      (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 600
 601         if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
 602             wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
 603                 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
 604         else if (wr->opcode != IB_WR_REG_MR)
 605                 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
 606                                         &curr_idx, valid_num_sge);
 607
 608         /*
 609          * The pipeline can sequentially post all valid WQEs into WQ buffer,
 610          * including new WQEs waiting for the doorbell to update the PI again.
 611          * Therefore, the owner bit of WQE MUST be updated after all fields
 612          * and extSGEs have been written into DDR instead of cache.
 613          */
 614         if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
 615                 dma_wmb();
 616
 617         *sge_idx = curr_idx;
 618         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
 619                      owner_bit);
 620
 621         return ret;
 622 }
 623
 624 static inline void update_sq_db(struct hns_roce_dev *hr_dev,
 625                                 struct hns_roce_qp *qp)
 626 {
 627         if (unlikely(qp->state == IB_QPS_ERR)) {
 628                 flush_cqe(hr_dev, qp);
 629         } else {
 630                 struct hns_roce_v2_db sq_db = {};
 631
 632                 hr_reg_write(&sq_db, DB_TAG, qp->doorbell_qpn);
 633                 hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB);
 634                 hr_reg_write(&sq_db, DB_PI, qp->sq.head);
 635                 hr_reg_write(&sq_db, DB_SL, qp->sl);
 636
 637                 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg);
 638         }
 639 }
 640
 641 static inline void update_rq_db(struct hns_roce_dev *hr_dev,
 642                                 struct hns_roce_qp *qp)
 643 {
 644         if (unlikely(qp->state == IB_QPS_ERR)) {
 645                 flush_cqe(hr_dev, qp);
 646         } else {
 647                 if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) {
 648                         *qp->rdb.db_record =
 649                                         qp->rq.head & V2_DB_PRODUCER_IDX_M;
 650                 } else {
 651                         struct hns_roce_v2_db rq_db = {};
 652
 653                         hr_reg_write(&rq_db, DB_TAG, qp->qpn);
 654                         hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);
 655                         hr_reg_write(&rq_db, DB_PI, qp->rq.head);
 656
 657                         hns_roce_write64(hr_dev, (__le32 *)&rq_db,
 658                                          qp->rq.db_reg);
 659                 }
 660         }
 661 }
 662
 663 static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val,
 664                               u64 __iomem *dest)
 665 {
 666 #define HNS_ROCE_WRITE_TIMES 8
 667         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 668         struct hnae3_handle *handle = priv->handle;
 669         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 670         int i;
 671
 672         if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle))
 673                 for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++)
 674                         writeq_relaxed(*(val + i), dest + i);
 675 }
 676
 677 static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
 678                        void *wqe)
 679 {
 680         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 681
 682         /* All kinds of DirectWQE have the same header field layout */
 683         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FLAG_S, 1);
 684         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_L_M,
 685                        V2_RC_SEND_WQE_BYTE_4_DB_SL_L_S, qp->sl);
 686         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_H_M,
 687                        V2_RC_SEND_WQE_BYTE_4_DB_SL_H_S, qp->sl >> 2);
 688         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_M,
 689                        V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head);
 690
 691         hns_roce_write512(hr_dev, wqe, qp->sq.db_reg);
 692 }
 693
 694 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 695                                  const struct ib_send_wr *wr,
 696                                  const struct ib_send_wr **bad_wr)
 697 {
 698         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 699         struct ib_device *ibdev = &hr_dev->ib_dev;
 700         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 701         unsigned long flags = 0;
 702         unsigned int owner_bit;
 703         unsigned int sge_idx;
 704         unsigned int wqe_idx;
 705         void *wqe = NULL;
 706         u32 nreq;
 707         int ret;
 708
 709         spin_lock_irqsave(&qp->sq.lock, flags);
 710
 711         ret = check_send_valid(hr_dev, qp);
 712         if (unlikely(ret)) {
 713                 *bad_wr = wr;
 714                 nreq = 0;
 715                 goto out;
 716         }
 717
 718         sge_idx = qp->next_sge;
 719
 720         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 721                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
 722                         ret = -ENOMEM;
 723                         *bad_wr = wr;
 724                         goto out;
 725                 }
 726
 727                 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
 728
 729                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
 730                         ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
 731                                   wr->num_sge, qp->sq.max_gs);
 732                         ret = -EINVAL;
 733                         *bad_wr = wr;
 734                         goto out;
 735                 }
 736
 737                 wqe = hns_roce_get_send_wqe(qp, wqe_idx);
 738                 qp->sq.wrid[wqe_idx] = wr->wr_id;
 739                 owner_bit =
 740                        ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
 741
 742                 /* Corresponding to the QP type, wqe process separately */
 743                 if (ibqp->qp_type == IB_QPT_RC)
 744                         ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 745                 else
 746                         ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 747
 748                 if (unlikely(ret)) {
 749                         *bad_wr = wr;
 750                         goto out;
 751                 }
 752         }
 753
 754 out:
 755         if (likely(nreq)) {
 756                 qp->sq.head += nreq;
 757                 qp->next_sge = sge_idx;
 758
 759                 if (nreq == 1 && (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
 760                         write_dwqe(hr_dev, qp, wqe);
 761                 else
 762                         update_sq_db(hr_dev, qp);
 763         }
 764
 765         spin_unlock_irqrestore(&qp->sq.lock, flags);
 766
 767         return ret;
 768 }
 769
 770 static int check_recv_valid(struct hns_roce_dev *hr_dev,
 771                             struct hns_roce_qp *hr_qp)
 772 {
 773         struct ib_device *ibdev = &hr_dev->ib_dev;
 774         struct ib_qp *ibqp = &hr_qp->ibqp;
 775
 776         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
 777                      ibqp->qp_type != IB_QPT_GSI &&
 778                      ibqp->qp_type != IB_QPT_UD)) {
 779                 ibdev_err(ibdev, "unsupported qp type, qp_type = %d.\n",
 780                           ibqp->qp_type);
 781                 return -EOPNOTSUPP;
 782         }
 783
 784         if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
 785                 return -EIO;
 786
 787         if (hr_qp->state == IB_QPS_RESET)
 788                 return -EINVAL;
 789
 790         return 0;
 791 }
 792
 793 static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe,
 794                                  u32 max_sge, bool rsv)
 795 {
 796         struct hns_roce_v2_wqe_data_seg *dseg = wqe;
 797         u32 i, cnt;
 798
 799         for (i = 0, cnt = 0; i < wr->num_sge; i++) {
 800                 /* Skip zero-length sge */
 801                 if (!wr->sg_list[i].length)
 802                         continue;
 803                 set_data_seg_v2(dseg + cnt, wr->sg_list + i);
 804                 cnt++;
 805         }
 806
 807         /* Fill a reserved sge to make hw stop reading remaining segments */
 808         if (rsv) {
 809                 dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 810                 dseg[cnt].addr = 0;
 811                 dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);
 812         } else {
 813                 /* Clear remaining segments to make ROCEE ignore sges */
 814                 if (cnt < max_sge)
 815                         memset(dseg + cnt, 0,
 816                                (max_sge - cnt) * HNS_ROCE_SGE_SIZE);
 817         }
 818 }
 819
 820 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,
 821                         u32 wqe_idx, u32 max_sge)
 822 {
 823         struct hns_roce_rinl_sge *sge_list;
 824         void *wqe = NULL;
 825         u32 i;
 826
 827         wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
 828         fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);
 829
 830         /* rq support inline data */
 831         if (hr_qp->rq_inl_buf.wqe_cnt) {
 832                 sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
 833                 hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge;
 834                 for (i = 0; i < wr->num_sge; i++) {
 835                         sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
 836                         sge_list[i].len = wr->sg_list[i].length;
 837                 }
 838         }
 839 }
 840
 841 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 842                                  const struct ib_recv_wr *wr,
 843                                  const struct ib_recv_wr **bad_wr)
 844 {
 845         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 846         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 847         struct ib_device *ibdev = &hr_dev->ib_dev;
 848         u32 wqe_idx, nreq, max_sge;
 849         unsigned long flags;
 850         int ret;
 851
 852         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 853
 854         ret = check_recv_valid(hr_dev, hr_qp);
 855         if (unlikely(ret)) {
 856                 *bad_wr = wr;
 857                 nreq = 0;
 858                 goto out;
 859         }
 860
 861         max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
 862         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 863                 if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
 864                                                   hr_qp->ibqp.recv_cq))) {
 865                         ret = -ENOMEM;
 866                         *bad_wr = wr;
 867                         goto out;
 868                 }
 869
 870                 if (unlikely(wr->num_sge > max_sge)) {
 871                         ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
 872                                   wr->num_sge, max_sge);
 873                         ret = -EINVAL;
 874                         *bad_wr = wr;
 875                         goto out;
 876                 }
 877
 878                 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
 879                 fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge);
 880                 hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
 881         }
 882
 883 out:
 884         if (likely(nreq)) {
 885                 hr_qp->rq.head += nreq;
 886
 887                 update_rq_db(hr_dev, hr_qp);
 888         }
 889         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 890
 891         return ret;
 892 }
 893
 894 static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n)
 895 {
 896         return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
 897 }
 898
 899 static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n)
 900 {
 901         return hns_roce_buf_offset(idx_que->mtr.kmem,
 902                                    n << idx_que->entry_shift);
 903 }
 904
 905 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index)
 906 {
 907         /* always called with interrupts disabled. */
 908         spin_lock(&srq->lock);
 909
 910         bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
 911         srq->idx_que.tail++;
 912
 913         spin_unlock(&srq->lock);
 914 }
 915
 916 static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq)
 917 {
 918         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 919
 920         return idx_que->head - idx_que->tail >= srq->wqe_cnt;
 921 }
 922
 923 static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge,
 924                                 const struct ib_recv_wr *wr)
 925 {
 926         struct ib_device *ib_dev = srq->ibsrq.device;
 927
 928         if (unlikely(wr->num_sge > max_sge)) {
 929                 ibdev_err(ib_dev,
 930                           "failed to check sge, wr->num_sge = %d, max_sge = %u.\n",
 931                           wr->num_sge, max_sge);
 932                 return -EINVAL;
 933         }
 934
 935         if (unlikely(hns_roce_srqwq_overflow(srq))) {
 936                 ibdev_err(ib_dev,
 937                           "failed to check srqwq status, srqwq is full.\n");
 938                 return -ENOMEM;
 939         }
 940
 941         return 0;
 942 }
 943
 944 static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx)
 945 {
 946         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 947         u32 pos;
 948
 949         pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt);
 950         if (unlikely(pos == srq->wqe_cnt))
 951                 return -ENOSPC;
 952
 953         bitmap_set(idx_que->bitmap, pos, 1);
 954         *wqe_idx = pos;
 955         return 0;
 956 }
 957
 958 static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)
 959 {
 960         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 961         unsigned int head;
 962         __le32 *buf;
 963
 964         head = idx_que->head & (srq->wqe_cnt - 1);
 965
 966         buf = get_idx_buf(idx_que, head);
 967         *buf = cpu_to_le32(wqe_idx);
 968
 969         idx_que->head++;
 970 }
 971
 972 static void update_srq_db(struct hns_roce_v2_db *db, struct hns_roce_srq *srq)
 973 {
 974         hr_reg_write(db, DB_TAG, srq->srqn);
 975         hr_reg_write(db, DB_CMD, HNS_ROCE_V2_SRQ_DB);
 976         hr_reg_write(db, DB_PI, srq->idx_que.head);
 977 }
 978
 979 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
 980                                      const struct ib_recv_wr *wr,
 981                                      const struct ib_recv_wr **bad_wr)
 982 {
 983         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
 984         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
 985         struct hns_roce_v2_db srq_db;
 986         unsigned long flags;
 987         int ret = 0;
 988         u32 max_sge;
 989         u32 wqe_idx;
 990         void *wqe;
 991         u32 nreq;
 992
 993         spin_lock_irqsave(&srq->lock, flags);
 994
 995         max_sge = srq->max_gs - srq->rsv_sge;
 996         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 997                 ret = check_post_srq_valid(srq, max_sge, wr);
 998                 if (ret) {
 999                         *bad_wr = wr;
1000                         break;
1001                 }
1002
1003                 ret = get_srq_wqe_idx(srq, &wqe_idx);
1004                 if (unlikely(ret)) {
1005                         *bad_wr = wr;
1006                         break;
1007                 }
1008
1009                 wqe = get_srq_wqe_buf(srq, wqe_idx);
1010                 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge);
1011                 fill_wqe_idx(srq, wqe_idx);
1012                 srq->wrid[wqe_idx] = wr->wr_id;
1013         }
1014
1015         if (likely(nreq)) {
1016                 update_srq_db(&srq_db, srq);
1017
1018                 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg);
1019         }
1020
1021         spin_unlock_irqrestore(&srq->lock, flags);
1022
1023         return ret;
1024 }
1025
1026 static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
1027                                       unsigned long instance_stage,
1028                                       unsigned long reset_stage)
1029 {
1030         /* When hardware reset has been completed once or more, we should stop
1031          * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
1032          * function, we should exit with error. If now at HNAE3_INIT_CLIENT
1033          * stage of soft reset process, we should exit with error, and then
1034          * HNAE3_INIT_CLIENT related process can rollback the operation like
1035          * notifing hardware to free resources, HNAE3_INIT_CLIENT related
1036          * process will exit with error to notify NIC driver to reschedule soft
1037          * reset process once again.
1038          */
1039         hr_dev->is_reset = true;
1040         hr_dev->dis_db = true;
1041
1042         if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
1043             instance_stage == HNS_ROCE_STATE_INIT)
1044                 return CMD_RST_PRC_EBUSY;
1045
1046         return CMD_RST_PRC_SUCCESS;
1047 }
1048
1049 static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
1050                                         unsigned long instance_stage,
1051                                         unsigned long reset_stage)
1052 {
1053         struct hns_roce_v2_priv *priv = hr_dev->priv;
1054         struct hnae3_handle *handle = priv->handle;
1055         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1056
1057         /* When hardware reset is detected, we should stop sending mailbox&cmq&
1058          * doorbell to hardware. If now in .init_instance() function, we should
1059          * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
1060          * process, we should exit with error, and then HNAE3_INIT_CLIENT
1061          * related process can rollback the operation like notifing hardware to
1062          * free resources, HNAE3_INIT_CLIENT related process will exit with
1063          * error to notify NIC driver to reschedule soft reset process once
1064          * again.
1065          */
1066         hr_dev->dis_db = true;
1067         if (!ops->get_hw_reset_stat(handle))
1068                 hr_dev->is_reset = true;
1069
1070         if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
1071             instance_stage == HNS_ROCE_STATE_INIT)
1072                 return CMD_RST_PRC_EBUSY;
1073
1074         return CMD_RST_PRC_SUCCESS;
1075 }
1076
1077 static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
1078 {
1079         struct hns_roce_v2_priv *priv = hr_dev->priv;
1080         struct hnae3_handle *handle = priv->handle;
1081         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1082
1083         /* When software reset is detected at .init_instance() function, we
1084          * should stop sending mailbox&cmq&doorbell to hardware, and exit
1085          * with error.
1086          */
1087         hr_dev->dis_db = true;
1088         if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
1089                 hr_dev->is_reset = true;
1090
1091         return CMD_RST_PRC_EBUSY;
1092 }
1093
1094 static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev,
1095                                     struct hnae3_handle *handle)
1096 {
1097         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1098         unsigned long instance_stage; /* the current instance stage */
1099         unsigned long reset_stage; /* the current reset stage */
1100         unsigned long reset_cnt;
1101         bool sw_resetting;
1102         bool hw_resetting;
1103
1104         /* Get information about reset from NIC driver or RoCE driver itself,
1105          * the meaning of the following variables from NIC driver are described
1106          * as below:
1107          * reset_cnt -- The count value of completed hardware reset.
1108          * hw_resetting -- Whether hardware device is resetting now.
1109          * sw_resetting -- Whether NIC's software reset process is running now.
1110          */
1111         instance_stage = handle->rinfo.instance_state;
1112         reset_stage = handle->rinfo.reset_state;
1113         reset_cnt = ops->ae_dev_reset_cnt(handle);
1114         if (reset_cnt != hr_dev->reset_cnt)
1115                 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
1116                                                   reset_stage);
1117
1118         hw_resetting = ops->get_cmdq_stat(handle);
1119         if (hw_resetting)
1120                 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
1121                                                     reset_stage);
1122
1123         sw_resetting = ops->ae_dev_resetting(handle);
1124         if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
1125                 return hns_roce_v2_cmd_sw_resetting(hr_dev);
1126
1127         return CMD_RST_PRC_OTHERS;
1128 }
1129
1130 static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev)
1131 {
1132         struct hns_roce_v2_priv *priv = hr_dev->priv;
1133         struct hnae3_handle *handle = priv->handle;
1134         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1135
1136         if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle))
1137                 return true;
1138
1139         if (ops->get_hw_reset_stat(handle))
1140                 return true;
1141
1142         if (ops->ae_dev_resetting(handle))
1143                 return true;
1144
1145         return false;
1146 }
1147
1148 static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy)
1149 {
1150         struct hns_roce_v2_priv *priv = hr_dev->priv;
1151         u32 status;
1152
1153         if (hr_dev->is_reset)
1154                 status = CMD_RST_PRC_SUCCESS;
1155         else
1156                 status = check_aedev_reset_status(hr_dev, priv->handle);
1157
1158         *busy = (status == CMD_RST_PRC_EBUSY);
1159
1160         return status == CMD_RST_PRC_OTHERS;
1161 }
1162
1163 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
1164                                    struct hns_roce_v2_cmq_ring *ring)
1165 {
1166         int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
1167
1168         ring->desc = kzalloc(size, GFP_KERNEL);
1169         if (!ring->desc)
1170                 return -ENOMEM;
1171
1172         ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
1173                                              DMA_BIDIRECTIONAL);
1174         if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
1175                 ring->desc_dma_addr = 0;
1176                 kfree(ring->desc);
1177                 ring->desc = NULL;
1178
1179                 return -ENOMEM;
1180         }
1181
1182         return 0;
1183 }
1184
1185 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
1186                                    struct hns_roce_v2_cmq_ring *ring)
1187 {
1188         dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
1189                          ring->desc_num * sizeof(struct hns_roce_cmq_desc),
1190                          DMA_BIDIRECTIONAL);
1191
1192         ring->desc_dma_addr = 0;
1193         kfree(ring->desc);
1194 }
1195
1196 static int init_csq(struct hns_roce_dev *hr_dev,
1197                     struct hns_roce_v2_cmq_ring *csq)
1198 {
1199         dma_addr_t dma;
1200         int ret;
1201
1202         csq->desc_num = CMD_CSQ_DESC_NUM;
1203         spin_lock_init(&csq->lock);
1204         csq->flag = TYPE_CSQ;
1205         csq->head = 0;
1206
1207         ret = hns_roce_alloc_cmq_desc(hr_dev, csq);
1208         if (ret)
1209                 return ret;
1210
1211         dma = csq->desc_dma_addr;
1212         roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma));
1213         roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma));
1214         roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
1215                    (u32)csq->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1216
1217         /* Make sure to write CI first and then PI */
1218         roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0);
1219         roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0);
1220
1221         return 0;
1222 }
1223
1224 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
1225 {
1226         struct hns_roce_v2_priv *priv = hr_dev->priv;
1227         int ret;
1228
1229         priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
1230
1231         ret = init_csq(hr_dev, &priv->cmq.csq);
1232         if (ret)
1233                 dev_err(hr_dev->dev, "failed to init CSQ, ret = %d.\n", ret);
1234
1235         return ret;
1236 }
1237
1238 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
1239 {
1240         struct hns_roce_v2_priv *priv = hr_dev->priv;
1241
1242         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
1243 }
1244
1245 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
1246                                           enum hns_roce_opcode_type opcode,
1247                                           bool is_read)
1248 {
1249         memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
1250         desc->opcode = cpu_to_le16(opcode);
1251         desc->flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);
1252         if (is_read)
1253                 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
1254         else
1255                 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1256 }
1257
1258 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
1259 {
1260         u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1261         struct hns_roce_v2_priv *priv = hr_dev->priv;
1262
1263         return tail == priv->cmq.csq.head;
1264 }
1265
1266 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1267                                struct hns_roce_cmq_desc *desc, int num)
1268 {
1269         struct hns_roce_v2_priv *priv = hr_dev->priv;
1270         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1271         u32 timeout = 0;
1272         u16 desc_ret;
1273         u32 tail;
1274         int ret;
1275         int i;
1276
1277         spin_lock_bh(&csq->lock);
1278
1279         tail = csq->head;
1280
1281         for (i = 0; i < num; i++) {
1282                 csq->desc[csq->head++] = desc[i];
1283                 if (csq->head == csq->desc_num)
1284                         csq->head = 0;
1285         }
1286
1287         /* Write to hardware */
1288         roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq->head);
1289
1290         do {
1291                 if (hns_roce_cmq_csq_done(hr_dev))
1292                         break;
1293                 udelay(1);
1294         } while (++timeout < priv->cmq.tx_timeout);
1295
1296         if (hns_roce_cmq_csq_done(hr_dev)) {
1297                 for (ret = 0, i = 0; i < num; i++) {
1298                         /* check the result of hardware write back */
1299                         desc[i] = csq->desc[tail++];
1300                         if (tail == csq->desc_num)
1301                                 tail = 0;
1302
1303                         desc_ret = le16_to_cpu(desc[i].retval);
1304                         if (likely(desc_ret == CMD_EXEC_SUCCESS))
1305                                 continue;
1306
1307                         dev_err_ratelimited(hr_dev->dev,
1308                                             "Cmdq IO error, opcode = %x, return = %x\n",
1309                                             desc->opcode, desc_ret);
1310                         ret = -EIO;
1311                 }
1312         } else {
1313                 /* FW/HW reset or incorrect number of desc */
1314                 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1315                 dev_warn(hr_dev->dev, "CMDQ move tail from %d to %d\n",
1316                          csq->head, tail);
1317                 csq->head = tail;
1318
1319                 ret = -EAGAIN;
1320         }
1321
1322         spin_unlock_bh(&csq->lock);
1323
1324         return ret;
1325 }
1326
1327 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1328                              struct hns_roce_cmq_desc *desc, int num)
1329 {
1330         bool busy;
1331         int ret;
1332
1333         if (!v2_chk_mbox_is_avail(hr_dev, &busy))
1334                 return busy ? -EBUSY : 0;
1335
1336         ret = __hns_roce_cmq_send(hr_dev, desc, num);
1337         if (ret) {
1338                 if (!v2_chk_mbox_is_avail(hr_dev, &busy))
1339                         return busy ? -EBUSY : 0;
1340         }
1341
1342         return ret;
1343 }
1344
1345 static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
1346                                dma_addr_t base_addr, u16 op)
1347 {
1348         struct hns_roce_cmd_mailbox *mbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1349         int ret;
1350
1351         if (IS_ERR(mbox))
1352                 return PTR_ERR(mbox);
1353
1354         ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, obj, 0, op,
1355                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
1356         hns_roce_free_cmd_mailbox(hr_dev, mbox);
1357         return ret;
1358 }
1359
1360 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1361 {
1362         struct hns_roce_query_version *resp;
1363         struct hns_roce_cmq_desc desc;
1364         int ret;
1365
1366         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1367         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1368         if (ret)
1369                 return ret;
1370
1371         resp = (struct hns_roce_query_version *)desc.data;
1372         hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
1373         hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1374
1375         return 0;
1376 }
1377
1378 static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev,
1379                                         struct hnae3_handle *handle)
1380 {
1381         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1382         unsigned long end;
1383
1384         hr_dev->dis_db = true;
1385
1386         dev_warn(hr_dev->dev,
1387                  "Func clear is pending, device in resetting state.\n");
1388         end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1389         while (end) {
1390                 if (!ops->get_hw_reset_stat(handle)) {
1391                         hr_dev->is_reset = true;
1392                         dev_info(hr_dev->dev,
1393                                  "Func clear success after reset.\n");
1394                         return;
1395                 }
1396                 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1397                 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1398         }
1399
1400         dev_warn(hr_dev->dev, "Func clear failed.\n");
1401 }
1402
1403 static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev,
1404                                         struct hnae3_handle *handle)
1405 {
1406         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1407         unsigned long end;
1408
1409         hr_dev->dis_db = true;
1410
1411         dev_warn(hr_dev->dev,
1412                  "Func clear is pending, device in resetting state.\n");
1413         end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1414         while (end) {
1415                 if (ops->ae_dev_reset_cnt(handle) !=
1416                     hr_dev->reset_cnt) {
1417                         hr_dev->is_reset = true;
1418                         dev_info(hr_dev->dev,
1419                                  "Func clear success after sw reset\n");
1420                         return;
1421                 }
1422                 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1423                 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1424         }
1425
1426         dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
1427 }
1428
1429 static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval,
1430                                        int flag)
1431 {
1432         struct hns_roce_v2_priv *priv = hr_dev->priv;
1433         struct hnae3_handle *handle = priv->handle;
1434         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1435
1436         if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) {
1437                 hr_dev->dis_db = true;
1438                 hr_dev->is_reset = true;
1439                 dev_info(hr_dev->dev, "Func clear success after reset.\n");
1440                 return;
1441         }
1442
1443         if (ops->get_hw_reset_stat(handle)) {
1444                 func_clr_hw_resetting_state(hr_dev, handle);
1445                 return;
1446         }
1447
1448         if (ops->ae_dev_resetting(handle) &&
1449             handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) {
1450                 func_clr_sw_resetting_state(hr_dev, handle);
1451                 return;
1452         }
1453
1454         if (retval && !flag)
1455                 dev_warn(hr_dev->dev,
1456                          "Func clear read failed, ret = %d.\n", retval);
1457
1458         dev_warn(hr_dev->dev, "Func clear failed.\n");
1459 }
1460
1461 static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id)
1462 {
1463         bool fclr_write_fail_flag = false;
1464         struct hns_roce_func_clear *resp;
1465         struct hns_roce_cmq_desc desc;
1466         unsigned long end;
1467         int ret = 0;
1468
1469         if (check_device_is_in_reset(hr_dev))
1470                 goto out;
1471
1472         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1473         resp = (struct hns_roce_func_clear *)desc.data;
1474         resp->rst_funcid_en = cpu_to_le32(vf_id);
1475
1476         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1477         if (ret) {
1478                 fclr_write_fail_flag = true;
1479                 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1480                          ret);
1481                 goto out;
1482         }
1483
1484         msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1485         end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1486         while (end) {
1487                 if (check_device_is_in_reset(hr_dev))
1488                         goto out;
1489                 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1490                 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1491
1492                 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1493                                               true);
1494
1495                 resp->rst_funcid_en = cpu_to_le32(vf_id);
1496                 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1497                 if (ret)
1498                         continue;
1499
1500                 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1501                         if (vf_id == 0)
1502                                 hr_dev->is_reset = true;
1503                         return;
1504                 }
1505         }
1506
1507 out:
1508         hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag);
1509 }
1510
1511 static void hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id)
1512 {
1513         enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;
1514         struct hns_roce_cmq_desc desc[2];
1515         struct hns_roce_cmq_req *req_a;
1516
1517         req_a = (struct hns_roce_cmq_req *)desc[0].data;
1518         hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
1519         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1520         hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
1521         hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id);
1522         hns_roce_cmq_send(hr_dev, desc, 2);
1523 }
1524
1525 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1526 {
1527         int i;
1528
1529         for (i = hr_dev->func_num - 1; i >= 0; i--) {
1530                 __hns_roce_function_clear(hr_dev, i);
1531                 if (i != 0)
1532                         hns_roce_free_vf_resource(hr_dev, i);
1533         }
1534 }
1535
1536 static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev)
1537 {
1538         struct hns_roce_cmq_desc desc;
1539         int ret;
1540
1541         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO,
1542                                       false);
1543         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1544         if (ret)
1545                 ibdev_err(&hr_dev->ib_dev,
1546                           "failed to clear extended doorbell info, ret = %d.\n",
1547                           ret);
1548
1549         return ret;
1550 }
1551
1552 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1553 {
1554         struct hns_roce_query_fw_info *resp;
1555         struct hns_roce_cmq_desc desc;
1556         int ret;
1557
1558         hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1559         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1560         if (ret)
1561                 return ret;
1562
1563         resp = (struct hns_roce_query_fw_info *)desc.data;
1564         hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1565
1566         return 0;
1567 }
1568
1569 static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
1570 {
1571         struct hns_roce_cmq_desc desc;
1572         int ret;
1573
1574         if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) {
1575                 hr_dev->func_num = 1;
1576                 return 0;
1577         }
1578
1579         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
1580                                       true);
1581         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1582         if (ret) {
1583                 hr_dev->func_num = 1;
1584                 return ret;
1585         }
1586
1587         hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num);
1588         hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id);
1589
1590         return 0;
1591 }
1592
1593 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1594 {
1595         struct hns_roce_cmq_desc desc;
1596         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1597
1598         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1599                                       false);
1600
1601         hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, 0x3e8);
1602         hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT);
1603
1604         return hns_roce_cmq_send(hr_dev, &desc, 1);
1605 }
1606
1607 static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf)
1608 {
1609         struct hns_roce_cmq_desc desc[2];
1610         struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
1611         struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
1612         struct hns_roce_caps *caps = &hr_dev->caps;
1613         enum hns_roce_opcode_type opcode;
1614         u32 func_num;
1615         int ret;
1616
1617         if (is_vf) {
1618                 opcode = HNS_ROCE_OPC_QUERY_VF_RES;
1619                 func_num = 1;
1620         } else {
1621                 opcode = HNS_ROCE_OPC_QUERY_PF_RES;
1622                 func_num = hr_dev->func_num;
1623         }
1624
1625         hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true);
1626         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1627         hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true);
1628
1629         ret = hns_roce_cmq_send(hr_dev, desc, 2);
1630         if (ret)
1631                 return ret;
1632
1633         caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num;
1634         caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num;
1635         caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num;
1636         caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num;
1637         caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num;
1638         caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num;
1639         caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num;
1640         caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num;
1641
1642         if (is_vf) {
1643                 caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num;
1644                 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) /
1645                                                func_num;
1646         } else {
1647                 caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num;
1648                 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) /
1649                                                func_num;
1650         }
1651
1652         return 0;
1653 }
1654
1655 static int load_ext_cfg_caps(struct hns_roce_dev *hr_dev, bool is_vf)
1656 {
1657         struct hns_roce_cmq_desc desc;
1658         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1659         struct hns_roce_caps *caps = &hr_dev->caps;
1660         u32 func_num, qp_num;
1661         int ret;
1662
1663         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_EXT_CFG, true);
1664         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1665         if (ret)
1666                 return ret;
1667
1668         func_num = is_vf ? 1 : max_t(u32, 1, hr_dev->func_num);
1669         qp_num = hr_reg_read(req, EXT_CFG_QP_PI_NUM) / func_num;
1670         caps->num_pi_qps = round_down(qp_num, HNS_ROCE_QP_BANK_NUM);
1671
1672         qp_num = hr_reg_read(req, EXT_CFG_QP_NUM) / func_num;
1673         caps->num_qps = round_down(qp_num, HNS_ROCE_QP_BANK_NUM);
1674
1675         return 0;
1676 }
1677
1678 static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev)
1679 {
1680         struct hns_roce_cmq_desc desc;
1681         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1682         struct hns_roce_caps *caps = &hr_dev->caps;
1683         int ret;
1684
1685         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1686                                       true);
1687
1688         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1689         if (ret)
1690                 return ret;
1691
1692         caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM);
1693         caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM);
1694
1695         return 0;
1696 }
1697
1698 static int query_func_resource_caps(struct hns_roce_dev *hr_dev, bool is_vf)
1699 {
1700         struct device *dev = hr_dev->dev;
1701         int ret;
1702
1703         ret = load_func_res_caps(hr_dev, is_vf);
1704         if (ret) {
1705                 dev_err(dev, "failed to load res caps, ret = %d (%s).\n", ret,
1706                         is_vf ? "vf" : "pf");
1707                 return ret;
1708         }
1709
1710         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
1711                 ret = load_ext_cfg_caps(hr_dev, is_vf);
1712                 if (ret)
1713                         dev_err(dev, "failed to load ext cfg, ret = %d (%s).\n",
1714                                 ret, is_vf ? "vf" : "pf");
1715         }
1716
1717         return ret;
1718 }
1719
1720 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1721 {
1722         struct device *dev = hr_dev->dev;
1723         int ret;
1724
1725         ret = query_func_resource_caps(hr_dev, false);
1726         if (ret)
1727                 return ret;
1728
1729         ret = load_pf_timer_res_caps(hr_dev);
1730         if (ret)
1731                 dev_err(dev, "failed to load pf timer resource, ret = %d.\n",
1732                         ret);
1733
1734         return ret;
1735 }
1736
1737 static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev)
1738 {
1739         return query_func_resource_caps(hr_dev, true);
1740 }
1741
1742 static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1743                                           u32 vf_id)
1744 {
1745         struct hns_roce_vf_switch *swt;
1746         struct hns_roce_cmq_desc desc;
1747         int ret;
1748
1749         swt = (struct hns_roce_vf_switch *)desc.data;
1750         hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1751         swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1752         roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1753                        VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id);
1754         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1755         if (ret)
1756                 return ret;
1757
1758         desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);
1759         desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1760         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
1761         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
1762         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1763
1764         return hns_roce_cmq_send(hr_dev, &desc, 1);
1765 }
1766
1767 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev)
1768 {
1769         u32 vf_id;
1770         int ret;
1771
1772         for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) {
1773                 ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id);
1774                 if (ret)
1775                         return ret;
1776         }
1777         return 0;
1778 }
1779
1780 static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id)
1781 {
1782         struct hns_roce_cmq_desc desc[2];
1783         struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
1784         struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
1785         enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;
1786         struct hns_roce_caps *caps = &hr_dev->caps;
1787
1788         hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
1789         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1790         hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
1791
1792         hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id);
1793
1794         hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num);
1795         hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num);
1796         hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num);
1797         hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num);
1798         hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num);
1799         hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num);
1800         hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num);
1801         hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num);
1802         hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num);
1803         hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num);
1804         hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num);
1805         hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num);
1806         hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num);
1807         hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num);
1808
1809         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
1810                 hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num);
1811                 hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX,
1812                              vf_id * caps->gmv_bt_num);
1813         } else {
1814                 hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num);
1815                 hr_reg_write(r_b, FUNC_RES_B_SGID_IDX,
1816                              vf_id * caps->sgid_bt_num);
1817                 hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num);
1818                 hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX,
1819                              vf_id * caps->smac_bt_num);
1820         }
1821
1822         return hns_roce_cmq_send(hr_dev, desc, 2);
1823 }
1824
1825 static int config_vf_ext_resource(struct hns_roce_dev *hr_dev, u32 vf_id)
1826 {
1827         struct hns_roce_cmq_desc desc;
1828         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1829         struct hns_roce_caps *caps = &hr_dev->caps;
1830
1831         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_EXT_CFG, false);
1832
1833         hr_reg_write(req, EXT_CFG_VF_ID, vf_id);
1834
1835         hr_reg_write(req, EXT_CFG_QP_PI_NUM, caps->num_pi_qps);
1836         hr_reg_write(req, EXT_CFG_QP_PI_IDX, vf_id * caps->num_pi_qps);
1837         hr_reg_write(req, EXT_CFG_QP_NUM, caps->num_qps);
1838         hr_reg_write(req, EXT_CFG_QP_IDX, vf_id * caps->num_qps);
1839
1840         return hns_roce_cmq_send(hr_dev, &desc, 1);
1841 }
1842
1843 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1844 {
1845         u32 func_num = max_t(u32, 1, hr_dev->func_num);
1846         u32 vf_id;
1847         int ret;
1848
1849         for (vf_id = 0; vf_id < func_num; vf_id++) {
1850                 ret = config_vf_hem_resource(hr_dev, vf_id);
1851                 if (ret) {
1852                         dev_err(hr_dev->dev,
1853                                 "failed to config vf-%u hem res, ret = %d.\n",
1854                                 vf_id, ret);
1855                         return ret;
1856                 }
1857
1858                 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
1859                         ret = config_vf_ext_resource(hr_dev, vf_id);
1860                         if (ret) {
1861                                 dev_err(hr_dev->dev,
1862                                         "failed to config vf-%u ext res, ret = %d.\n",
1863                                         vf_id, ret);
1864                                 return ret;
1865                         }
1866                 }
1867         }
1868
1869         return 0;
1870 }
1871
1872 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1873 {
1874         struct hns_roce_cmq_desc desc;
1875         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1876         struct hns_roce_caps *caps = &hr_dev->caps;
1877
1878         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1879
1880         hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ,
1881                      caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1882         hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ,
1883                      caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1884         hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM,
1885                      to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps));
1886
1887         hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ,
1888                      caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1889         hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ,
1890                      caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1891         hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM,
1892                      to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs));
1893
1894         hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ,
1895                      caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1896         hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ,
1897                      caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1898         hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM,
1899                      to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs));
1900
1901         hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ,
1902                      caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1903         hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ,
1904                      caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1905         hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM,
1906                      to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts));
1907
1908         hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ,
1909                      caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1910         hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ,
1911                      caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1912         hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM,
1913                      to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps));
1914
1915         return hns_roce_cmq_send(hr_dev, &desc, 1);
1916 }
1917
1918 /* Use default caps when hns_roce_query_pf_caps() failed or init VF profile */
1919 static void set_default_caps(struct hns_roce_dev *hr_dev)
1920 {
1921         struct hns_roce_caps *caps = &hr_dev->caps;
1922
1923         caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
1924         caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
1925         caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
1926         caps->num_srqs          = HNS_ROCE_V2_MAX_SRQ_NUM;
1927         caps->min_cqes          = HNS_ROCE_MIN_CQE_NUM;
1928         caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
1929         caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1930         caps->max_extend_sg     = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1931         caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1932
1933         caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
1934         caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
1935         caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
1936         caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1937         caps->num_comp_vectors  = 0;
1938
1939         caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
1940         caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
1941         caps->num_qpc_timer     = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1942         caps->num_cqc_timer     = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1943
1944         caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1945         caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1946         caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1947         caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1948         caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1949         caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1950         caps->trrl_entry_sz     = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
1951         caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
1952         caps->srqc_entry_sz     = HNS_ROCE_V2_SRQC_ENTRY_SZ;
1953         caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1954         caps->idx_entry_sz      = HNS_ROCE_V2_IDX_ENTRY_SZ;
1955         caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1956         caps->reserved_lkey     = 0;
1957         caps->reserved_pds      = 0;
1958         caps->reserved_mrws     = 1;
1959         caps->reserved_uars     = 0;
1960         caps->reserved_cqs      = 0;
1961         caps->reserved_srqs     = 0;
1962         caps->reserved_qps      = HNS_ROCE_V2_RSV_QPS;
1963
1964         caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1965         caps->srqc_hop_num      = HNS_ROCE_CONTEXT_HOP_NUM;
1966         caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1967         caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1968         caps->sccc_hop_num      = HNS_ROCE_SCCC_HOP_NUM;
1969
1970         caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
1971         caps->wqe_sq_hop_num    = HNS_ROCE_SQWQE_HOP_NUM;
1972         caps->wqe_sge_hop_num   = HNS_ROCE_EXT_SGE_HOP_NUM;
1973         caps->wqe_rq_hop_num    = HNS_ROCE_RQWQE_HOP_NUM;
1974         caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
1975         caps->srqwqe_hop_num    = HNS_ROCE_SRQWQE_HOP_NUM;
1976         caps->idx_hop_num       = HNS_ROCE_IDX_HOP_NUM;
1977         caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1978
1979         caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
1980                                   HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1981                                   HNS_ROCE_CAP_FLAG_CQ_RECORD_DB |
1982                                   HNS_ROCE_CAP_FLAG_QP_RECORD_DB;
1983
1984         caps->pkey_table_len[0] = 1;
1985         caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
1986         caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
1987         caps->local_ca_ack_delay = 0;
1988         caps->max_mtu = IB_MTU_4096;
1989
1990         caps->max_srq_wrs       = HNS_ROCE_V2_MAX_SRQ_WR;
1991         caps->max_srq_sges      = HNS_ROCE_V2_MAX_SRQ_SGE;
1992
1993         caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
1994                        HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
1995                        HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL | HNS_ROCE_CAP_FLAG_XRC;
1996
1997         caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1998
1999         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2000                 caps->flags |= HNS_ROCE_CAP_FLAG_STASH;
2001                 caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE;
2002         } else {
2003                 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
2004
2005                 /* The following configuration are only valid for HIP08 */
2006                 caps->qpc_sz = HNS_ROCE_V2_QPC_SZ;
2007                 caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ;
2008                 caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE;
2009         }
2010 }
2011
2012 static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,
2013                        u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)
2014 {
2015         u64 obj_per_chunk;
2016         u64 bt_chunk_size = PAGE_SIZE;
2017         u64 buf_chunk_size = PAGE_SIZE;
2018         u64 obj_per_chunk_default = buf_chunk_size / obj_size;
2019
2020         *buf_page_size = 0;
2021         *bt_page_size = 0;
2022
2023         switch (hop_num) {
2024         case 3:
2025                 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
2026                                 (bt_chunk_size / BA_BYTE_LEN) *
2027                                 (bt_chunk_size / BA_BYTE_LEN) *
2028                                  obj_per_chunk_default;
2029                 break;
2030         case 2:
2031                 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
2032                                 (bt_chunk_size / BA_BYTE_LEN) *
2033                                  obj_per_chunk_default;
2034                 break;
2035         case 1:
2036                 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
2037                                 obj_per_chunk_default;
2038                 break;
2039         case HNS_ROCE_HOP_NUM_0:
2040                 obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
2041                 break;
2042         default:
2043                 pr_err("table %u not support hop_num = %u!\n", hem_type,
2044                        hop_num);
2045                 return;
2046         }
2047
2048         if (hem_type >= HEM_TYPE_MTT)
2049                 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
2050         else
2051                 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
2052 }
2053
2054 static void set_hem_page_size(struct hns_roce_dev *hr_dev)
2055 {
2056         struct hns_roce_caps *caps = &hr_dev->caps;
2057
2058         /* EQ */
2059         caps->eqe_ba_pg_sz = 0;
2060         caps->eqe_buf_pg_sz = 0;
2061
2062         /* Link Table */
2063         caps->llm_buf_pg_sz = 0;
2064
2065         /* MR */
2066         caps->mpt_ba_pg_sz = 0;
2067         caps->mpt_buf_pg_sz = 0;
2068         caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
2069         caps->pbl_buf_pg_sz = 0;
2070         calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
2071                    caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
2072                    HEM_TYPE_MTPT);
2073
2074         /* QP */
2075         caps->qpc_ba_pg_sz = 0;
2076         caps->qpc_buf_pg_sz = 0;
2077         caps->qpc_timer_ba_pg_sz = 0;
2078         caps->qpc_timer_buf_pg_sz = 0;
2079         caps->sccc_ba_pg_sz = 0;
2080         caps->sccc_buf_pg_sz = 0;
2081         caps->mtt_ba_pg_sz = 0;
2082         caps->mtt_buf_pg_sz = 0;
2083         calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
2084                    caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
2085                    HEM_TYPE_QPC);
2086
2087         if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
2088                 calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num,
2089                            caps->sccc_bt_num, &caps->sccc_buf_pg_sz,
2090                            &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC);
2091
2092         /* CQ */
2093         caps->cqc_ba_pg_sz = 0;
2094         caps->cqc_buf_pg_sz = 0;
2095         caps->cqc_timer_ba_pg_sz = 0;
2096         caps->cqc_timer_buf_pg_sz = 0;
2097         caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
2098         caps->cqe_buf_pg_sz = 0;
2099         calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
2100                    caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
2101                    HEM_TYPE_CQC);
2102         calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num,
2103                    1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
2104
2105         /* SRQ */
2106         if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) {
2107                 caps->srqc_ba_pg_sz = 0;
2108                 caps->srqc_buf_pg_sz = 0;
2109                 caps->srqwqe_ba_pg_sz = 0;
2110                 caps->srqwqe_buf_pg_sz = 0;
2111                 caps->idx_ba_pg_sz = 0;
2112                 caps->idx_buf_pg_sz = 0;
2113                 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz,
2114                            caps->srqc_hop_num, caps->srqc_bt_num,
2115                            &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz,
2116                            HEM_TYPE_SRQC);
2117                 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
2118                            caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
2119                            &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
2120                 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz,
2121                            caps->idx_hop_num, 1, &caps->idx_buf_pg_sz,
2122                            &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
2123         }
2124
2125         /* GMV */
2126         caps->gmv_ba_pg_sz = 0;
2127         caps->gmv_buf_pg_sz = 0;
2128 }
2129
2130 /* Apply all loaded caps before setting to hardware */
2131 static void apply_func_caps(struct hns_roce_dev *hr_dev)
2132 {
2133         struct hns_roce_caps *caps = &hr_dev->caps;
2134         struct hns_roce_v2_priv *priv = hr_dev->priv;
2135
2136         /* The following configurations don't need to be got from firmware. */
2137         caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
2138         caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
2139         caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
2140
2141         caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
2142         caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
2143         caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2144         caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2145
2146         caps->num_xrcds = HNS_ROCE_V2_MAX_XRCD_NUM;
2147         caps->reserved_xrcds = HNS_ROCE_V2_RSV_XRCD_NUM;
2148
2149         caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
2150         caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
2151         caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
2152
2153         if (!caps->num_comp_vectors)
2154                 caps->num_comp_vectors = min_t(u32, caps->eqc_bt_num - 1,
2155                                   (u32)priv->handle->rinfo.num_vectors - 2);
2156
2157         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2158                 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
2159                 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
2160
2161                 /* The following configurations will be overwritten */
2162                 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
2163                 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
2164                 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
2165
2166                 /* The following configurations are not got from firmware */
2167                 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
2168
2169                 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
2170                 caps->gid_table_len[0] = caps->gmv_bt_num *
2171                                         (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz);
2172
2173                 caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /
2174                                                           caps->gmv_entry_sz);
2175         } else {
2176                 u32 func_num = max_t(u32, 1, hr_dev->func_num);
2177
2178                 caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
2179                 caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
2180                 caps->gid_table_len[0] /= func_num;
2181         }
2182
2183         if (hr_dev->is_vf) {
2184                 caps->default_aeq_arm_st = 0x3;
2185                 caps->default_ceq_arm_st = 0x3;
2186                 caps->default_ceq_max_cnt = 0x1;
2187                 caps->default_ceq_period = 0x10;
2188                 caps->default_aeq_max_cnt = 0x1;
2189                 caps->default_aeq_period = 0x10;
2190         }
2191
2192         set_hem_page_size(hr_dev);
2193 }
2194
2195 static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
2196 {
2197         struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
2198         struct hns_roce_caps *caps = &hr_dev->caps;
2199         struct hns_roce_query_pf_caps_a *resp_a;
2200         struct hns_roce_query_pf_caps_b *resp_b;
2201         struct hns_roce_query_pf_caps_c *resp_c;
2202         struct hns_roce_query_pf_caps_d *resp_d;
2203         struct hns_roce_query_pf_caps_e *resp_e;
2204         int ctx_hop_num;
2205         int pbl_hop_num;
2206         int ret;
2207         int i;
2208
2209         for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
2210                 hns_roce_cmq_setup_basic_desc(&desc[i],
2211                                               HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
2212                                               true);
2213                 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
2214                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2215                 else
2216                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2217         }
2218
2219         ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
2220         if (ret)
2221                 return ret;
2222
2223         resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
2224         resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
2225         resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
2226         resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
2227         resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
2228
2229         caps->local_ca_ack_delay     = resp_a->local_ca_ack_delay;
2230         caps->max_sq_sg              = le16_to_cpu(resp_a->max_sq_sg);
2231         caps->max_sq_inline          = le16_to_cpu(resp_a->max_sq_inline);
2232         caps->max_rq_sg              = le16_to_cpu(resp_a->max_rq_sg);
2233         caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg);
2234         caps->max_extend_sg          = le32_to_cpu(resp_a->max_extend_sg);
2235         caps->num_qpc_timer          = le16_to_cpu(resp_a->num_qpc_timer);
2236         caps->num_cqc_timer          = le16_to_cpu(resp_a->num_cqc_timer);
2237         caps->max_srq_sges           = le16_to_cpu(resp_a->max_srq_sges);
2238         caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges);
2239         caps->num_aeq_vectors        = resp_a->num_aeq_vectors;
2240         caps->num_other_vectors      = resp_a->num_other_vectors;
2241         caps->max_sq_desc_sz         = resp_a->max_sq_desc_sz;
2242         caps->max_rq_desc_sz         = resp_a->max_rq_desc_sz;
2243         caps->max_srq_desc_sz        = resp_a->max_srq_desc_sz;
2244         caps->cqe_sz                 = resp_a->cqe_sz;
2245
2246         caps->mtpt_entry_sz          = resp_b->mtpt_entry_sz;
2247         caps->irrl_entry_sz          = resp_b->irrl_entry_sz;
2248         caps->trrl_entry_sz          = resp_b->trrl_entry_sz;
2249         caps->cqc_entry_sz           = resp_b->cqc_entry_sz;
2250         caps->srqc_entry_sz          = resp_b->srqc_entry_sz;
2251         caps->idx_entry_sz           = resp_b->idx_entry_sz;
2252         caps->sccc_sz                = resp_b->sccc_sz;
2253         caps->max_mtu                = resp_b->max_mtu;
2254         caps->qpc_sz                 = le16_to_cpu(resp_b->qpc_sz);
2255         caps->min_cqes               = resp_b->min_cqes;
2256         caps->min_wqes               = resp_b->min_wqes;
2257         caps->page_size_cap          = le32_to_cpu(resp_b->page_size_cap);
2258         caps->pkey_table_len[0]      = resp_b->pkey_table_len;
2259         caps->phy_num_uars           = resp_b->phy_num_uars;
2260         ctx_hop_num                  = resp_b->ctx_hop_num;
2261         pbl_hop_num                  = resp_b->pbl_hop_num;
2262
2263         caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds,
2264                                             V2_QUERY_PF_CAPS_C_NUM_PDS_M,
2265                                             V2_QUERY_PF_CAPS_C_NUM_PDS_S);
2266         caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
2267                                      V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
2268                                      V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
2269         caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
2270                        HNS_ROCE_CAP_FLAGS_EX_SHIFT;
2271
2272         caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
2273                                             V2_QUERY_PF_CAPS_C_NUM_CQS_M,
2274                                             V2_QUERY_PF_CAPS_C_NUM_CQS_S);
2275         caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs,
2276                                                 V2_QUERY_PF_CAPS_C_MAX_GID_M,
2277                                                 V2_QUERY_PF_CAPS_C_MAX_GID_S);
2278
2279         caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth,
2280                                              V2_QUERY_PF_CAPS_C_CQ_DEPTH_M,
2281                                              V2_QUERY_PF_CAPS_C_CQ_DEPTH_S);
2282         caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws,
2283                                               V2_QUERY_PF_CAPS_C_NUM_MRWS_M,
2284                                               V2_QUERY_PF_CAPS_C_NUM_MRWS_S);
2285         caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps,
2286                                             V2_QUERY_PF_CAPS_C_NUM_QPS_M,
2287                                             V2_QUERY_PF_CAPS_C_NUM_QPS_S);
2288         caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps,
2289                                                 V2_QUERY_PF_CAPS_C_MAX_ORD_M,
2290                                                 V2_QUERY_PF_CAPS_C_MAX_ORD_S);
2291         caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
2292         caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
2293         caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs,
2294                                              V2_QUERY_PF_CAPS_D_NUM_SRQS_M,
2295                                              V2_QUERY_PF_CAPS_D_NUM_SRQS_S);
2296         caps->cong_type = roce_get_field(resp_d->wq_hop_num_max_srqs,
2297                                          V2_QUERY_PF_CAPS_D_CONG_TYPE_M,
2298                                          V2_QUERY_PF_CAPS_D_CONG_TYPE_S);
2299         caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
2300
2301         caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth,
2302                                                V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M,
2303                                                V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S);
2304         caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth,
2305                                                 V2_QUERY_PF_CAPS_D_NUM_CEQS_M,
2306                                                 V2_QUERY_PF_CAPS_D_NUM_CEQS_S);
2307
2308         caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth,
2309                                                V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M,
2310                                                V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S);
2311         caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
2312                                             V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M,
2313                                             V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S);
2314         caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
2315                                             V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M,
2316                                             V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S);
2317         caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds,
2318                                             V2_QUERY_PF_CAPS_D_RSV_PDS_M,
2319                                             V2_QUERY_PF_CAPS_D_RSV_PDS_S);
2320         caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds,
2321                                              V2_QUERY_PF_CAPS_D_NUM_UARS_M,
2322                                              V2_QUERY_PF_CAPS_D_NUM_UARS_S);
2323         caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps,
2324                                             V2_QUERY_PF_CAPS_D_RSV_QPS_M,
2325                                             V2_QUERY_PF_CAPS_D_RSV_QPS_S);
2326         caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps,
2327                                              V2_QUERY_PF_CAPS_D_RSV_UARS_M,
2328                                              V2_QUERY_PF_CAPS_D_RSV_UARS_S);
2329         caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
2330                                              V2_QUERY_PF_CAPS_E_RSV_MRWS_M,
2331                                              V2_QUERY_PF_CAPS_E_RSV_MRWS_S);
2332         caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
2333                                          V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M,
2334                                          V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S);
2335         caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs,
2336                                             V2_QUERY_PF_CAPS_E_RSV_CQS_M,
2337                                             V2_QUERY_PF_CAPS_E_RSV_CQS_S);
2338         caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs,
2339                                              V2_QUERY_PF_CAPS_E_RSV_SRQS_M,
2340                                              V2_QUERY_PF_CAPS_E_RSV_SRQS_S);
2341         caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey,
2342                                              V2_QUERY_PF_CAPS_E_RSV_LKEYS_M,
2343                                              V2_QUERY_PF_CAPS_E_RSV_LKEYS_S);
2344         caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
2345         caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
2346         caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
2347         caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
2348
2349         caps->qpc_hop_num = ctx_hop_num;
2350         caps->sccc_hop_num = ctx_hop_num;
2351         caps->srqc_hop_num = ctx_hop_num;
2352         caps->cqc_hop_num = ctx_hop_num;
2353         caps->mpt_hop_num = ctx_hop_num;
2354         caps->mtt_hop_num = pbl_hop_num;
2355         caps->cqe_hop_num = pbl_hop_num;
2356         caps->srqwqe_hop_num = pbl_hop_num;
2357         caps->idx_hop_num = pbl_hop_num;
2358         caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
2359                                           V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M,
2360                                           V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S);
2361         caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
2362                                           V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M,
2363                                           V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S);
2364         caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
2365                                           V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M,
2366                                           V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S);
2367
2368         return 0;
2369 }
2370
2371 static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val)
2372 {
2373         struct hns_roce_cmq_desc desc;
2374         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
2375
2376         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
2377                                       false);
2378
2379         hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type);
2380         hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val);
2381
2382         return hns_roce_cmq_send(hr_dev, &desc, 1);
2383 }
2384
2385 static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)
2386 {
2387         struct hns_roce_caps *caps = &hr_dev->caps;
2388         int ret;
2389
2390         if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
2391                 return 0;
2392
2393         ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,
2394                                     caps->qpc_sz);
2395         if (ret) {
2396                 dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);
2397                 return ret;
2398         }
2399
2400         ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE,
2401                                     caps->sccc_sz);
2402         if (ret)
2403                 dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);
2404
2405         return ret;
2406 }
2407
2408 static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev)
2409 {
2410         struct device *dev = hr_dev->dev;
2411         int ret;
2412
2413         hr_dev->func_num = 1;
2414
2415         set_default_caps(hr_dev);
2416
2417         ret = hns_roce_query_vf_resource(hr_dev);
2418         if (ret) {
2419                 dev_err(dev, "failed to query VF resource, ret = %d.\n", ret);
2420                 return ret;
2421         }
2422
2423         apply_func_caps(hr_dev);
2424
2425         ret = hns_roce_v2_set_bt(hr_dev);
2426         if (ret)
2427                 dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret);
2428
2429         return ret;
2430 }
2431
2432 static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev)
2433 {
2434         struct device *dev = hr_dev->dev;
2435         int ret;
2436
2437         ret = hns_roce_query_func_info(hr_dev);
2438         if (ret) {
2439                 dev_err(dev, "failed to query func info, ret = %d.\n", ret);
2440                 return ret;
2441         }
2442
2443         ret = hns_roce_config_global_param(hr_dev);
2444         if (ret) {
2445                 dev_err(dev, "failed to config global param, ret = %d.\n", ret);
2446                 return ret;
2447         }
2448
2449         ret = hns_roce_set_vf_switch_param(hr_dev);
2450         if (ret) {
2451                 dev_err(dev, "failed to set switch param, ret = %d.\n", ret);
2452                 return ret;
2453         }
2454
2455         ret = hns_roce_query_pf_caps(hr_dev);
2456         if (ret)
2457                 set_default_caps(hr_dev);
2458
2459         ret = hns_roce_query_pf_resource(hr_dev);
2460         if (ret) {
2461                 dev_err(dev, "failed to query pf resource, ret = %d.\n", ret);
2462                 return ret;
2463         }
2464
2465         apply_func_caps(hr_dev);
2466
2467         ret = hns_roce_alloc_vf_resource(hr_dev);
2468         if (ret) {
2469                 dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret);
2470                 return ret;
2471         }
2472
2473         ret = hns_roce_v2_set_bt(hr_dev);
2474         if (ret) {
2475                 dev_err(dev, "failed to config BA table, ret = %d.\n", ret);
2476                 return ret;
2477         }
2478
2479         /* Configure the size of QPC, SCCC, etc. */
2480         return hns_roce_config_entry_size(hr_dev);
2481 }
2482
2483 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
2484 {
2485         struct device *dev = hr_dev->dev;
2486         int ret;
2487
2488         ret = hns_roce_cmq_query_hw_info(hr_dev);
2489         if (ret) {
2490                 dev_err(dev, "failed to query hardware info, ret = %d.\n", ret);
2491                 return ret;
2492         }
2493
2494         ret = hns_roce_query_fw_ver(hr_dev);
2495         if (ret) {
2496                 dev_err(dev, "failed to query firmware info, ret = %d.\n", ret);
2497                 return ret;
2498         }
2499
2500         hr_dev->vendor_part_id = hr_dev->pci_dev->device;
2501         hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
2502
2503         if (hr_dev->is_vf)
2504                 return hns_roce_v2_vf_profile(hr_dev);
2505         else
2506                 return hns_roce_v2_pf_profile(hr_dev);
2507 }
2508
2509 static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf)
2510 {
2511         u32 i, next_ptr, page_num;
2512         __le64 *entry = cfg_buf;
2513         dma_addr_t addr;
2514         u64 val;
2515
2516         page_num = data_buf->npages;
2517         for (i = 0; i < page_num; i++) {
2518                 addr = hns_roce_buf_page(data_buf, i);
2519                 if (i == (page_num - 1))
2520                         next_ptr = 0;
2521                 else
2522                         next_ptr = i + 1;
2523
2524                 val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr);
2525                 entry[i] = cpu_to_le64(val);
2526         }
2527 }
2528
2529 static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev,
2530                              struct hns_roce_link_table *table)
2531 {
2532         struct hns_roce_cmq_desc desc[2];
2533         struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
2534         struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
2535         struct hns_roce_buf *buf = table->buf;
2536         enum hns_roce_opcode_type opcode;
2537         dma_addr_t addr;
2538
2539         opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
2540         hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
2541         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2542         hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
2543
2544         hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map));
2545         hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map));
2546         hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages);
2547         hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf->page_shift));
2548         hr_reg_enable(r_a, CFG_LLM_A_INIT_EN);
2549
2550         addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0));
2551         hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr));
2552         hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr));
2553         hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1);
2554         hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0);
2555
2556         addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf->npages - 1));
2557         hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr));
2558         hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr));
2559         hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1);
2560
2561         return hns_roce_cmq_send(hr_dev, desc, 2);
2562 }
2563
2564 static struct hns_roce_link_table *
2565 alloc_link_table_buf(struct hns_roce_dev *hr_dev)
2566 {
2567         struct hns_roce_v2_priv *priv = hr_dev->priv;
2568         struct hns_roce_link_table *link_tbl;
2569         u32 pg_shift, size, min_size;
2570
2571         link_tbl = &priv->ext_llm;
2572         pg_shift = hr_dev->caps.llm_buf_pg_sz + PAGE_SHIFT;
2573         size = hr_dev->caps.num_qps * HNS_ROCE_V2_EXT_LLM_ENTRY_SZ;
2574         min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(hr_dev->caps.sl_num) << pg_shift;
2575
2576         /* Alloc data table */
2577         size = max(size, min_size);
2578         link_tbl->buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0);
2579         if (IS_ERR(link_tbl->buf))
2580                 return ERR_PTR(-ENOMEM);
2581
2582         /* Alloc config table */
2583         size = link_tbl->buf->npages * sizeof(u64);
2584         link_tbl->table.buf = dma_alloc_coherent(hr_dev->dev, size,
2585                                                  &link_tbl->table.map,
2586                                                  GFP_KERNEL);
2587         if (!link_tbl->table.buf) {
2588                 hns_roce_buf_free(hr_dev, link_tbl->buf);
2589                 return ERR_PTR(-ENOMEM);
2590         }
2591
2592         return link_tbl;
2593 }
2594
2595 static void free_link_table_buf(struct hns_roce_dev *hr_dev,
2596                                 struct hns_roce_link_table *tbl)
2597 {
2598         if (tbl->buf) {
2599                 u32 size = tbl->buf->npages * sizeof(u64);
2600
2601                 dma_free_coherent(hr_dev->dev, size, tbl->table.buf,
2602                                   tbl->table.map);
2603         }
2604
2605         hns_roce_buf_free(hr_dev, tbl->buf);
2606 }
2607
2608 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev)
2609 {
2610         struct hns_roce_link_table *link_tbl;
2611         int ret;
2612
2613         link_tbl = alloc_link_table_buf(hr_dev);
2614         if (IS_ERR(link_tbl))
2615                 return -ENOMEM;
2616
2617         if (WARN_ON(link_tbl->buf->npages > HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) {
2618                 ret = -EINVAL;
2619                 goto err_alloc;
2620         }
2621
2622         config_llm_table(link_tbl->buf, link_tbl->table.buf);
2623         ret = set_llm_cfg_to_hw(hr_dev, link_tbl);
2624         if (ret)
2625                 goto err_alloc;
2626
2627         return 0;
2628
2629 err_alloc:
2630         free_link_table_buf(hr_dev, link_tbl);
2631         return ret;
2632 }
2633
2634 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev)
2635 {
2636         struct hns_roce_v2_priv *priv = hr_dev->priv;
2637
2638         free_link_table_buf(hr_dev, &priv->ext_llm);
2639 }
2640
2641 static void free_dip_list(struct hns_roce_dev *hr_dev)
2642 {
2643         struct hns_roce_dip *hr_dip;
2644         struct hns_roce_dip *tmp;
2645         unsigned long flags;
2646
2647         spin_lock_irqsave(&hr_dev->dip_list_lock, flags);
2648
2649         list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) {
2650                 list_del(&hr_dip->node);
2651                 kfree(hr_dip);
2652         }
2653
2654         spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);
2655 }
2656
2657 static int get_hem_table(struct hns_roce_dev *hr_dev)
2658 {
2659         unsigned int qpc_count;
2660         unsigned int cqc_count;
2661         unsigned int gmv_count;
2662         int ret;
2663         int i;
2664
2665         /* Alloc memory for source address table buffer space chunk */
2666         for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;
2667              gmv_count++) {
2668                 ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);
2669                 if (ret)
2670                         goto err_gmv_failed;
2671         }
2672
2673         if (hr_dev->is_vf)
2674                 return 0;
2675
2676         /* Alloc memory for QPC Timer buffer space chunk */
2677         for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
2678              qpc_count++) {
2679                 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
2680                                          qpc_count);
2681                 if (ret) {
2682                         dev_err(hr_dev->dev, "QPC Timer get failed\n");
2683                         goto err_qpc_timer_failed;
2684                 }
2685         }
2686
2687         /* Alloc memory for CQC Timer buffer space chunk */
2688         for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
2689              cqc_count++) {
2690                 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
2691                                          cqc_count);
2692                 if (ret) {
2693                         dev_err(hr_dev->dev, "CQC Timer get failed\n");
2694                         goto err_cqc_timer_failed;
2695                 }
2696         }
2697
2698         return 0;
2699
2700 err_cqc_timer_failed:
2701         for (i = 0; i < cqc_count; i++)
2702                 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2703
2704 err_qpc_timer_failed:
2705         for (i = 0; i < qpc_count; i++)
2706                 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2707
2708 err_gmv_failed:
2709         for (i = 0; i < gmv_count; i++)
2710                 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
2711
2712         return ret;
2713 }
2714
2715 static void put_hem_table(struct hns_roce_dev *hr_dev)
2716 {
2717         int i;
2718
2719         for (i = 0; i < hr_dev->caps.gmv_entry_num; i++)
2720                 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
2721
2722         if (hr_dev->is_vf)
2723                 return;
2724
2725         for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++)
2726                 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2727
2728         for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++)
2729                 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2730 }
2731
2732 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
2733 {
2734         int ret;
2735
2736         /* The hns ROCEE requires the extdb info to be cleared before using */
2737         ret = hns_roce_clear_extdb_list_info(hr_dev);
2738         if (ret)
2739                 return ret;
2740
2741         ret = get_hem_table(hr_dev);
2742         if (ret)
2743                 return ret;
2744
2745         if (hr_dev->is_vf)
2746                 return 0;
2747
2748         ret = hns_roce_init_link_table(hr_dev);
2749         if (ret) {
2750                 dev_err(hr_dev->dev, "failed to init llm, ret = %d.\n", ret);
2751                 goto err_llm_init_failed;
2752         }
2753
2754         return 0;
2755
2756 err_llm_init_failed:
2757         put_hem_table(hr_dev);
2758
2759         return ret;
2760 }
2761
2762 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
2763 {
2764         hns_roce_function_clear(hr_dev);
2765
2766         if (!hr_dev->is_vf)
2767                 hns_roce_free_link_table(hr_dev);
2768
2769         if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09)
2770                 free_dip_list(hr_dev);
2771 }
2772
2773 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
2774                               u64 out_param, u32 in_modifier, u8 op_modifier,
2775                               u16 op, u16 token, int event)
2776 {
2777         struct hns_roce_cmq_desc desc;
2778         struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
2779
2780         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
2781
2782         mb->in_param_l = cpu_to_le32(in_param);
2783         mb->in_param_h = cpu_to_le32(in_param >> 32);
2784         mb->out_param_l = cpu_to_le32(out_param);
2785         mb->out_param_h = cpu_to_le32(out_param >> 32);
2786         mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
2787         mb->token_event_en = cpu_to_le32(event << 16 | token);
2788
2789         return hns_roce_cmq_send(hr_dev, &desc, 1);
2790 }
2791
2792 static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout,
2793                                  u8 *complete_status)
2794 {
2795         struct hns_roce_mbox_status *mb_st;
2796         struct hns_roce_cmq_desc desc;
2797         unsigned long end;
2798         int ret = -EBUSY;
2799         u32 status;
2800         bool busy;
2801
2802         mb_st = (struct hns_roce_mbox_status *)desc.data;
2803         end = msecs_to_jiffies(timeout) + jiffies;
2804         while (v2_chk_mbox_is_avail(hr_dev, &busy)) {
2805                 status = 0;
2806                 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST,
2807                                               true);
2808                 ret = __hns_roce_cmq_send(hr_dev, &desc, 1);
2809                 if (!ret) {
2810                         status = le32_to_cpu(mb_st->mb_status_hw_run);
2811                         /* No pending message exists in ROCEE mbox. */
2812                         if (!(status & MB_ST_HW_RUN_M))
2813                                 break;
2814                 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {
2815                         break;
2816                 }
2817
2818                 if (time_after(jiffies, end)) {
2819                         dev_err_ratelimited(hr_dev->dev,
2820                                             "failed to wait mbox status 0x%x\n",
2821                                             status);
2822                         return -ETIMEDOUT;
2823                 }
2824
2825                 cond_resched();
2826                 ret = -EBUSY;
2827         }
2828
2829         if (!ret) {
2830                 *complete_status = (u8)(status & MB_ST_COMPLETE_M);
2831         } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {
2832                 /* Ignore all errors if the mbox is unavailable. */
2833                 ret = 0;
2834                 *complete_status = MB_ST_COMPLETE_M;
2835         }
2836
2837         return ret;
2838 }
2839
2840 static int v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
2841                         u64 out_param, u32 in_modifier, u8 op_modifier,
2842                         u16 op, u16 token, int event)
2843 {
2844         u8 status = 0;
2845         int ret;
2846
2847         /* Waiting for the mbox to be idle */
2848         ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS,
2849                                     &status);
2850         if (unlikely(ret)) {
2851                 dev_err_ratelimited(hr_dev->dev,
2852                                     "failed to check post mbox status = 0x%x, ret = %d.\n",
2853                                     status, ret);
2854                 return ret;
2855         }
2856
2857         /* Post new message to mbox */
2858         ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2859                                  op_modifier, op, token, event);
2860         if (ret)
2861                 dev_err_ratelimited(hr_dev->dev,
2862                                     "failed to post mailbox, ret = %d.\n", ret);
2863
2864         return ret;
2865 }
2866
2867 static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev, unsigned int timeout)
2868 {
2869         u8 status = 0;
2870         int ret;
2871
2872         ret = v2_wait_mbox_complete(hr_dev, timeout, &status);
2873         if (!ret) {
2874                 if (status != MB_ST_COMPLETE_SUCC)
2875                         return -EBUSY;
2876         } else {
2877                 dev_err_ratelimited(hr_dev->dev,
2878                                     "failed to check mbox status = 0x%x, ret = %d.\n",
2879                                     status, ret);
2880         }
2881
2882         return ret;
2883 }
2884
2885 static void copy_gid(void *dest, const union ib_gid *gid)
2886 {
2887 #define GID_SIZE 4
2888         const union ib_gid *src = gid;
2889         __le32 (*p)[GID_SIZE] = dest;
2890         int i;
2891
2892         if (!gid)
2893                 src = &zgid;
2894
2895         for (i = 0; i < GID_SIZE; i++)
2896                 (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]);
2897 }
2898
2899 static int config_sgid_table(struct hns_roce_dev *hr_dev,
2900                              int gid_index, const union ib_gid *gid,
2901                              enum hns_roce_sgid_type sgid_type)
2902 {
2903         struct hns_roce_cmq_desc desc;
2904         struct hns_roce_cfg_sgid_tb *sgid_tb =
2905                                     (struct hns_roce_cfg_sgid_tb *)desc.data;
2906
2907         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2908
2909         roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M,
2910                        CFG_SGID_TB_TABLE_IDX_S, gid_index);
2911         roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
2912                        CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2913
2914         copy_gid(&sgid_tb->vf_sgid_l, gid);
2915
2916         return hns_roce_cmq_send(hr_dev, &desc, 1);
2917 }
2918
2919 static int config_gmv_table(struct hns_roce_dev *hr_dev,
2920                             int gid_index, const union ib_gid *gid,
2921                             enum hns_roce_sgid_type sgid_type,
2922                             const struct ib_gid_attr *attr)
2923 {
2924         struct hns_roce_cmq_desc desc[2];
2925         struct hns_roce_cfg_gmv_tb_a *tb_a =
2926                                 (struct hns_roce_cfg_gmv_tb_a *)desc[0].data;
2927         struct hns_roce_cfg_gmv_tb_b *tb_b =
2928                                 (struct hns_roce_cfg_gmv_tb_b *)desc[1].data;
2929
2930         u16 vlan_id = VLAN_CFI_MASK;
2931         u8 mac[ETH_ALEN] = {};
2932         int ret;
2933
2934         if (gid) {
2935                 ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac);
2936                 if (ret)
2937                         return ret;
2938         }
2939
2940         hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);
2941         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2942
2943         hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);
2944
2945         copy_gid(&tb_a->vf_sgid_l, gid);
2946
2947         roce_set_field(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_SGID_TYPE_M,
2948                        CFG_GMV_TB_VF_SGID_TYPE_S, sgid_type);
2949         roce_set_bit(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_VLAN_EN_S,
2950                      vlan_id < VLAN_CFI_MASK);
2951         roce_set_field(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_VLAN_ID_M,
2952                        CFG_GMV_TB_VF_VLAN_ID_S, vlan_id);
2953
2954         tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac);
2955         roce_set_field(tb_b->vf_smac_h, CFG_GMV_TB_SMAC_H_M,
2956                        CFG_GMV_TB_SMAC_H_S, *(u16 *)&mac[4]);
2957
2958         roce_set_field(tb_b->table_idx_rsv, CFG_GMV_TB_SGID_IDX_M,
2959                        CFG_GMV_TB_SGID_IDX_S, gid_index);
2960
2961         return hns_roce_cmq_send(hr_dev, desc, 2);
2962 }
2963
2964 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u32 port,
2965                                int gid_index, const union ib_gid *gid,
2966                                const struct ib_gid_attr *attr)
2967 {
2968         enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
2969         int ret;
2970
2971         if (gid) {
2972                 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2973                         if (ipv6_addr_v4mapped((void *)gid))
2974                                 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2975                         else
2976                                 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2977                 } else if (attr->gid_type == IB_GID_TYPE_ROCE) {
2978                         sgid_type = GID_TYPE_FLAG_ROCE_V1;
2979                 }
2980         }
2981
2982         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
2983                 ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);
2984         else
2985                 ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2986
2987         if (ret)
2988                 ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n",
2989                           ret);
2990
2991         return ret;
2992 }
2993
2994 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2995                                u8 *addr)
2996 {
2997         struct hns_roce_cmq_desc desc;
2998         struct hns_roce_cfg_smac_tb *smac_tb =
2999                                     (struct hns_roce_cfg_smac_tb *)desc.data;
3000         u16 reg_smac_h;
3001         u32 reg_smac_l;
3002
3003         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
3004
3005         reg_smac_l = *(u32 *)(&addr[0]);
3006         reg_smac_h = *(u16 *)(&addr[4]);
3007
3008         roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M,
3009                        CFG_SMAC_TB_IDX_S, phy_port);
3010         roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M,
3011                        CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
3012         smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
3013
3014         return hns_roce_cmq_send(hr_dev, &desc, 1);
3015 }
3016
3017 static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
3018                         struct hns_roce_v2_mpt_entry *mpt_entry,
3019                         struct hns_roce_mr *mr)
3020 {
3021         u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
3022         struct ib_device *ibdev = &hr_dev->ib_dev;
3023         dma_addr_t pbl_ba;
3024         int i, count;
3025
3026         count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
3027                                   ARRAY_SIZE(pages), &pbl_ba);
3028         if (count < 1) {
3029                 ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
3030                           count);
3031                 return -ENOBUFS;
3032         }
3033
3034         /* Aligned to the hardware address access unit */
3035         for (i = 0; i < count; i++)
3036                 pages[i] >>= 6;
3037
3038         mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3039         mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
3040         roce_set_field(mpt_entry->byte_48_mode_ba,
3041                        V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
3042                        upper_32_bits(pbl_ba >> 3));
3043
3044         mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
3045         roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
3046                        V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
3047
3048         mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
3049         roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
3050                        V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
3051         roce_set_field(mpt_entry->byte_64_buf_pa1,
3052                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
3053                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
3054                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3055
3056         return 0;
3057 }
3058
3059 static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
3060                                   void *mb_buf, struct hns_roce_mr *mr,
3061                                   unsigned long mtpt_idx)
3062 {
3063         struct hns_roce_v2_mpt_entry *mpt_entry;
3064         int ret;
3065
3066         mpt_entry = mb_buf;
3067         memset(mpt_entry, 0, sizeof(*mpt_entry));
3068
3069         hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
3070         hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3071         hr_reg_enable(mpt_entry, MPT_L_INV_EN);
3072
3073         hr_reg_write_bool(mpt_entry, MPT_BIND_EN,
3074                           mr->access & IB_ACCESS_MW_BIND);
3075         hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN,
3076                           mr->access & IB_ACCESS_REMOTE_ATOMIC);
3077         hr_reg_write_bool(mpt_entry, MPT_RR_EN,
3078                           mr->access & IB_ACCESS_REMOTE_READ);
3079         hr_reg_write_bool(mpt_entry, MPT_RW_EN,
3080                           mr->access & IB_ACCESS_REMOTE_WRITE);
3081         hr_reg_write_bool(mpt_entry, MPT_LW_EN,
3082                           mr->access & IB_ACCESS_LOCAL_WRITE);
3083
3084         mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
3085         mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
3086         mpt_entry->lkey = cpu_to_le32(mr->key);
3087         mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
3088         mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
3089
3090         if (mr->type != MR_TYPE_MR)
3091                 hr_reg_enable(mpt_entry, MPT_PA);
3092
3093         if (mr->type == MR_TYPE_DMA)
3094                 return 0;
3095
3096         if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0)
3097                 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num);
3098
3099         hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
3100                      to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
3101         hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD);
3102
3103         ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
3104
3105         return ret;
3106 }
3107
3108 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
3109                                         struct hns_roce_mr *mr, int flags,
3110                                         void *mb_buf)
3111 {
3112         struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
3113         u32 mr_access_flags = mr->access;
3114         int ret = 0;
3115
3116         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
3117                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
3118
3119         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
3120                        V2_MPT_BYTE_4_PD_S, mr->pd);
3121
3122         if (flags & IB_MR_REREG_ACCESS) {
3123                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
3124                              V2_MPT_BYTE_8_BIND_EN_S,
3125                              (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
3126                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
3127                              V2_MPT_BYTE_8_ATOMIC_EN_S,
3128                              mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
3129                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
3130                              mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
3131                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
3132                              mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
3133                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
3134                              mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
3135         }
3136
3137         if (flags & IB_MR_REREG_TRANS) {
3138                 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
3139                 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
3140                 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
3141                 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
3142
3143                 ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
3144         }
3145
3146         return ret;
3147 }
3148
3149 static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev,
3150                                        void *mb_buf, struct hns_roce_mr *mr)
3151 {
3152         struct ib_device *ibdev = &hr_dev->ib_dev;
3153         struct hns_roce_v2_mpt_entry *mpt_entry;
3154         dma_addr_t pbl_ba = 0;
3155
3156         mpt_entry = mb_buf;
3157         memset(mpt_entry, 0, sizeof(*mpt_entry));
3158
3159         if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
3160                 ibdev_err(ibdev, "failed to find frmr mtr.\n");
3161                 return -ENOBUFS;
3162         }
3163
3164         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
3165                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
3166         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
3167                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
3168         roce_set_field(mpt_entry->byte_4_pd_hop_st,
3169                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
3170                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
3171                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
3172         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
3173                        V2_MPT_BYTE_4_PD_S, mr->pd);
3174
3175         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
3176         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
3177         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
3178
3179         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
3180         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
3181         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
3182         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
3183
3184         mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3185
3186         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3));
3187         roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
3188                        V2_MPT_BYTE_48_PBL_BA_H_S,
3189                        upper_32_bits(pbl_ba >> 3));
3190
3191         roce_set_field(mpt_entry->byte_64_buf_pa1,
3192                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
3193                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
3194                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3195
3196         return 0;
3197 }
3198
3199 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
3200 {
3201         struct hns_roce_v2_mpt_entry *mpt_entry;
3202
3203         mpt_entry = mb_buf;
3204         memset(mpt_entry, 0, sizeof(*mpt_entry));
3205
3206         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
3207                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
3208         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
3209                        V2_MPT_BYTE_4_PD_S, mw->pdn);
3210         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
3211                        V2_MPT_BYTE_4_PBL_HOP_NUM_S,
3212                        mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
3213                                                                mw->pbl_hop_num);
3214         roce_set_field(mpt_entry->byte_4_pd_hop_st,
3215                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
3216                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
3217                        mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
3218
3219         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
3220         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
3221         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 1);
3222
3223         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
3224         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
3225         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
3226         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
3227                      mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
3228
3229         roce_set_field(mpt_entry->byte_64_buf_pa1,
3230                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
3231                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
3232                        mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
3233
3234         mpt_entry->lkey = cpu_to_le32(mw->rkey);
3235
3236         return 0;
3237 }
3238
3239 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
3240 {
3241         return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
3242 }
3243
3244 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)
3245 {
3246         struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
3247
3248         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
3249         return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & hr_cq->cq_depth)) ? cqe :
3250                                                                          NULL;
3251 }
3252
3253 static inline void update_cq_db(struct hns_roce_dev *hr_dev,
3254                                 struct hns_roce_cq *hr_cq)
3255 {
3256         if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) {
3257                 *hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M;
3258         } else {
3259                 struct hns_roce_v2_db cq_db = {};
3260
3261                 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);
3262                 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB);
3263                 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);
3264                 hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1);
3265
3266                 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);
3267         }
3268 }
3269
3270 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3271                                    struct hns_roce_srq *srq)
3272 {
3273         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3274         struct hns_roce_v2_cqe *cqe, *dest;
3275         u32 prod_index;
3276         int nfreed = 0;
3277         int wqe_index;
3278         u8 owner_bit;
3279
3280         for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
3281              ++prod_index) {
3282                 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
3283                         break;
3284         }
3285
3286         /*
3287          * Now backwards through the CQ, removing CQ entries
3288          * that match our QP by overwriting them with next entries.
3289          */
3290         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
3291                 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
3292                 if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
3293                         if (srq && hr_reg_read(cqe, CQE_S_R)) {
3294                                 wqe_index = hr_reg_read(cqe, CQE_WQE_IDX);
3295                                 hns_roce_free_srq_wqe(srq, wqe_index);
3296                         }
3297                         ++nfreed;
3298                 } else if (nfreed) {
3299                         dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
3300                                           hr_cq->ib_cq.cqe);
3301                         owner_bit = hr_reg_read(dest, CQE_OWNER);
3302                         memcpy(dest, cqe, sizeof(*cqe));
3303                         hr_reg_write(dest, CQE_OWNER, owner_bit);
3304                 }
3305         }
3306
3307         if (nfreed) {
3308                 hr_cq->cons_index += nfreed;
3309                 update_cq_db(hr_dev, hr_cq);
3310         }
3311 }
3312
3313 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3314                                  struct hns_roce_srq *srq)
3315 {
3316         spin_lock_irq(&hr_cq->lock);
3317         __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
3318         spin_unlock_irq(&hr_cq->lock);
3319 }
3320
3321 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
3322                                   struct hns_roce_cq *hr_cq, void *mb_buf,
3323                                   u64 *mtts, dma_addr_t dma_handle)
3324 {
3325         struct hns_roce_v2_cq_context *cq_context;
3326
3327         cq_context = mb_buf;
3328         memset(cq_context, 0, sizeof(*cq_context));
3329
3330         hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID);
3331         hr_reg_write(cq_context, CQC_ARM_ST, REG_NXT_CEQE);
3332         hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth));
3333         hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector);
3334         hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn);
3335
3336         if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE)
3337                 hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B);
3338
3339         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3340                 hr_reg_enable(cq_context, CQC_STASH);
3341
3342         hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L,
3343                      to_hr_hw_page_addr(mtts[0]));
3344         hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H,
3345                      upper_32_bits(to_hr_hw_page_addr(mtts[0])));
3346         hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev->caps.cqe_hop_num ==
3347                      HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
3348         hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L,
3349                      to_hr_hw_page_addr(mtts[1]));
3350         hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H,
3351                      upper_32_bits(to_hr_hw_page_addr(mtts[1])));
3352         hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ,
3353                      to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
3354         hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ,
3355                      to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
3356         hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> 3);
3357         hr_reg_write(cq_context, CQC_CQE_BA_H, (dma_handle >> (32 + 3)));
3358         hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN,
3359                           hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB);
3360         hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L,
3361                      ((u32)hr_cq->db.dma) >> 1);
3362         hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H,
3363                      hr_cq->db.dma >> 32);
3364         hr_reg_write(cq_context, CQC_CQ_MAX_CNT,
3365                      HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
3366         hr_reg_write(cq_context, CQC_CQ_PERIOD,
3367                      HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
3368 }
3369
3370 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
3371                                      enum ib_cq_notify_flags flags)
3372 {
3373         struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3374         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3375         struct hns_roce_v2_db cq_db = {};
3376         u32 notify_flag;
3377
3378         /*
3379          * flags = 0, then notify_flag : next
3380          * flags = 1, then notify flag : solocited
3381          */
3382         notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
3383                       V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
3384
3385         hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);
3386         hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY);
3387         hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);
3388         hr_reg_write(&cq_db, DB_CQ_CMD_SN, hr_cq->arm_sn);
3389         hr_reg_write(&cq_db, DB_CQ_NOTIFY, notify_flag);
3390
3391         hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);
3392
3393         return 0;
3394 }
3395
3396 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
3397                                         struct hns_roce_qp *qp,
3398                                         struct ib_wc *wc)
3399 {
3400         struct hns_roce_rinl_sge *sge_list;
3401         u32 wr_num, wr_cnt, sge_num;
3402         u32 sge_cnt, data_len, size;
3403         void *wqe_buf;
3404
3405         wr_num = hr_reg_read(cqe, CQE_WQE_IDX);
3406         wr_cnt = wr_num & (qp->rq.wqe_cnt - 1);
3407
3408         sge_list = qp->rq_inl_buf.wqe_list[wr_cnt].sg_list;
3409         sge_num = qp->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
3410         wqe_buf = hns_roce_get_recv_wqe(qp, wr_cnt);
3411         data_len = wc->byte_len;
3412
3413         for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
3414                 size = min(sge_list[sge_cnt].len, data_len);
3415                 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
3416
3417                 data_len -= size;
3418                 wqe_buf += size;
3419         }
3420
3421         if (unlikely(data_len)) {
3422                 wc->status = IB_WC_LOC_LEN_ERR;
3423                 return -EAGAIN;
3424         }
3425
3426         return 0;
3427 }
3428
3429 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
3430                    int num_entries, struct ib_wc *wc)
3431 {
3432         unsigned int left;
3433         int npolled = 0;
3434
3435         left = wq->head - wq->tail;
3436         if (left == 0)
3437                 return 0;
3438
3439         left = min_t(unsigned int, (unsigned int)num_entries, left);
3440         while (npolled < left) {
3441                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3442                 wc->status = IB_WC_WR_FLUSH_ERR;
3443                 wc->vendor_err = 0;
3444                 wc->qp = &hr_qp->ibqp;
3445
3446                 wq->tail++;
3447                 wc++;
3448                 npolled++;
3449         }
3450
3451         return npolled;
3452 }
3453
3454 static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
3455                                   struct ib_wc *wc)
3456 {
3457         struct hns_roce_qp *hr_qp;
3458         int npolled = 0;
3459
3460         list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
3461                 npolled += sw_comp(hr_qp, &hr_qp->sq,
3462                                    num_entries - npolled, wc + npolled);
3463                 if (npolled >= num_entries)
3464                         goto out;
3465         }
3466
3467         list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
3468                 npolled += sw_comp(hr_qp, &hr_qp->rq,
3469                                    num_entries - npolled, wc + npolled);
3470                 if (npolled >= num_entries)
3471                         goto out;
3472         }
3473
3474 out:
3475         return npolled;
3476 }
3477
3478 static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
3479                            struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe,
3480                            struct ib_wc *wc)
3481 {
3482         static const struct {
3483                 u32 cqe_status;
3484                 enum ib_wc_status wc_status;
3485         } map[] = {
3486                 { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
3487                 { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
3488                 { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
3489                 { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
3490                 { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
3491                 { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
3492                 { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
3493                 { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
3494                 { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
3495                 { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
3496                 { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
3497                 { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
3498                   IB_WC_RETRY_EXC_ERR },
3499                 { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
3500                 { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
3501                 { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
3502         };
3503
3504         u32 cqe_status = hr_reg_read(cqe, CQE_STATUS);
3505         int i;
3506
3507         wc->status = IB_WC_GENERAL_ERR;
3508         for (i = 0; i < ARRAY_SIZE(map); i++)
3509                 if (cqe_status == map[i].cqe_status) {
3510                         wc->status = map[i].wc_status;
3511                         break;
3512                 }
3513
3514         if (likely(wc->status == IB_WC_SUCCESS ||
3515                    wc->status == IB_WC_WR_FLUSH_ERR))
3516                 return;
3517
3518         ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status);
3519         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,
3520                        cq->cqe_size, false);
3521         wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
3522
3523         /*
3524          * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
3525          * the standard protocol, the driver must ignore it and needn't to set
3526          * the QP to an error state.
3527          */
3528         if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
3529                 return;
3530
3531         flush_cqe(hr_dev, qp);
3532 }
3533
3534 static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe,
3535                       struct hns_roce_qp **cur_qp)
3536 {
3537         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3538         struct hns_roce_qp *hr_qp = *cur_qp;
3539         u32 qpn;
3540
3541         qpn = hr_reg_read(cqe, CQE_LCL_QPN);
3542
3543         if (!hr_qp || qpn != hr_qp->qpn) {
3544                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3545                 if (unlikely(!hr_qp)) {
3546                         ibdev_err(&hr_dev->ib_dev,
3547                                   "CQ %06lx with entry for unknown QPN %06x\n",
3548                                   hr_cq->cqn, qpn);
3549                         return -EINVAL;
3550                 }
3551                 *cur_qp = hr_qp;
3552         }
3553
3554         return 0;
3555 }
3556
3557 /*
3558  * mapped-value = 1 + real-value
3559  * The ib wc opcode's real value is start from 0, In order to distinguish
3560  * between initialized and uninitialized map values, we plus 1 to the actual
3561  * value when defining the mapping, so that the validity can be identified by
3562  * checking whether the mapped value is greater than 0.
3563  */
3564 #define HR_WC_OP_MAP(hr_key, ib_key) \
3565                 [HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key
3566
3567 static const u32 wc_send_op_map[] = {
3568         HR_WC_OP_MAP(SEND,                      SEND),
3569         HR_WC_OP_MAP(SEND_WITH_INV,             SEND),
3570         HR_WC_OP_MAP(SEND_WITH_IMM,             SEND),
3571         HR_WC_OP_MAP(RDMA_READ,                 RDMA_READ),
3572         HR_WC_OP_MAP(RDMA_WRITE,                RDMA_WRITE),
3573         HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM,       RDMA_WRITE),
3574         HR_WC_OP_MAP(LOCAL_INV,                 LOCAL_INV),
3575         HR_WC_OP_MAP(ATOM_CMP_AND_SWAP,         COMP_SWAP),
3576         HR_WC_OP_MAP(ATOM_FETCH_AND_ADD,        FETCH_ADD),
3577         HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP,     MASKED_COMP_SWAP),
3578         HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD,    MASKED_FETCH_ADD),
3579         HR_WC_OP_MAP(FAST_REG_PMR,              REG_MR),
3580         HR_WC_OP_MAP(BIND_MW,                   REG_MR),
3581 };
3582
3583 static int to_ib_wc_send_op(u32 hr_opcode)
3584 {
3585         if (hr_opcode >= ARRAY_SIZE(wc_send_op_map))
3586                 return -EINVAL;
3587
3588         return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 :
3589                                            -EINVAL;
3590 }
3591
3592 static const u32 wc_recv_op_map[] = {
3593         HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM,               WITH_IMM),
3594         HR_WC_OP_MAP(SEND,                              RECV),
3595         HR_WC_OP_MAP(SEND_WITH_IMM,                     WITH_IMM),
3596         HR_WC_OP_MAP(SEND_WITH_INV,                     RECV),
3597 };
3598
3599 static int to_ib_wc_recv_op(u32 hr_opcode)
3600 {
3601         if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map))
3602                 return -EINVAL;
3603
3604         return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 :
3605                                            -EINVAL;
3606 }
3607
3608 static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
3609 {
3610         u32 hr_opcode;
3611         int ib_opcode;
3612
3613         wc->wc_flags = 0;
3614
3615         hr_opcode = hr_reg_read(cqe, CQE_OPCODE);
3616         switch (hr_opcode) {
3617         case HNS_ROCE_V2_WQE_OP_RDMA_READ:
3618                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3619                 break;
3620         case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
3621         case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
3622                 wc->wc_flags |= IB_WC_WITH_IMM;
3623                 break;
3624         case HNS_ROCE_V2_WQE_OP_LOCAL_INV:
3625                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3626                 break;
3627         case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
3628         case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
3629         case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
3630         case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
3631                 wc->byte_len  = 8;
3632                 break;
3633         default:
3634                 break;
3635         }
3636
3637         ib_opcode = to_ib_wc_send_op(hr_opcode);
3638         if (ib_opcode < 0)
3639                 wc->status = IB_WC_GENERAL_ERR;
3640         else
3641                 wc->opcode = ib_opcode;
3642 }
3643
3644 static inline bool is_rq_inl_enabled(struct ib_wc *wc, u32 hr_opcode,
3645                                      struct hns_roce_v2_cqe *cqe)
3646 {
3647         return wc->qp->qp_type != IB_QPT_UD && wc->qp->qp_type != IB_QPT_GSI &&
3648                (hr_opcode == HNS_ROCE_V2_OPCODE_SEND ||
3649                 hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
3650                 hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
3651                hr_reg_read(cqe, CQE_RQ_INLINE);
3652 }
3653
3654 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
3655 {
3656         struct hns_roce_qp *qp = to_hr_qp(wc->qp);
3657         u32 hr_opcode;
3658         int ib_opcode;
3659         int ret;
3660
3661         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3662
3663         hr_opcode = hr_reg_read(cqe, CQE_OPCODE);
3664         switch (hr_opcode) {
3665         case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
3666         case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
3667                 wc->wc_flags = IB_WC_WITH_IMM;
3668                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata));
3669                 break;
3670         case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
3671                 wc->wc_flags = IB_WC_WITH_INVALIDATE;
3672                 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
3673                 break;
3674         default:
3675                 wc->wc_flags = 0;
3676         }
3677
3678         ib_opcode = to_ib_wc_recv_op(hr_opcode);
3679         if (ib_opcode < 0)
3680                 wc->status = IB_WC_GENERAL_ERR;
3681         else
3682                 wc->opcode = ib_opcode;
3683
3684         if (is_rq_inl_enabled(wc, hr_opcode, cqe)) {
3685                 ret = hns_roce_handle_recv_inl_wqe(cqe, qp, wc);
3686                 if (unlikely(ret))
3687                         return ret;
3688         }
3689
3690         wc->sl = hr_reg_read(cqe, CQE_SL);
3691         wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);
3692         wc->slid = 0;
3693         wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0;
3694         wc->port_num = hr_reg_read(cqe, CQE_PORTN);
3695         wc->pkey_index = 0;
3696
3697         if (hr_reg_read(cqe, CQE_VID_VLD)) {
3698                 wc->vlan_id = hr_reg_read(cqe, CQE_VID);
3699                 wc->wc_flags |= IB_WC_WITH_VLAN;
3700         } else {
3701                 wc->vlan_id = 0xffff;
3702         }
3703
3704         wc->network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE);
3705
3706         return 0;
3707 }
3708
3709 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
3710                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
3711 {
3712         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3713         struct hns_roce_qp *qp = *cur_qp;
3714         struct hns_roce_srq *srq = NULL;
3715         struct hns_roce_v2_cqe *cqe;
3716         struct hns_roce_wq *wq;
3717         int is_send;
3718         u16 wqe_idx;
3719         int ret;
3720
3721         cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
3722         if (!cqe)
3723                 return -EAGAIN;
3724
3725         ++hr_cq->cons_index;
3726         /* Memory barrier */
3727         rmb();
3728
3729         ret = get_cur_qp(hr_cq, cqe, &qp);
3730         if (ret)
3731                 return ret;
3732
3733         wc->qp = &qp->ibqp;
3734         wc->vendor_err = 0;
3735
3736         wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
3737
3738         is_send = !hr_reg_read(cqe, CQE_S_R);
3739         if (is_send) {
3740                 wq = &qp->sq;
3741
3742                 /* If sg_signal_bit is set, tail pointer will be updated to
3743                  * the WQE corresponding to the current CQE.
3744                  */
3745                 if (qp->sq_signal_bits)
3746                         wq->tail += (wqe_idx - (u16)wq->tail) &
3747                                     (wq->wqe_cnt - 1);
3748
3749                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3750                 ++wq->tail;
3751
3752                 fill_send_wc(wc, cqe);
3753         } else {
3754                 if (qp->ibqp.srq) {
3755                         srq = to_hr_srq(qp->ibqp.srq);
3756                         wc->wr_id = srq->wrid[wqe_idx];
3757                         hns_roce_free_srq_wqe(srq, wqe_idx);
3758                 } else {
3759                         wq = &qp->rq;
3760                         wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3761                         ++wq->tail;
3762                 }
3763
3764                 ret = fill_recv_wc(wc, cqe);
3765         }
3766
3767         get_cqe_status(hr_dev, qp, hr_cq, cqe, wc);
3768         if (unlikely(wc->status != IB_WC_SUCCESS))
3769                 return 0;
3770
3771         return ret;
3772 }
3773
3774 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
3775                                struct ib_wc *wc)
3776 {
3777         struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3778         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3779         struct hns_roce_qp *cur_qp = NULL;
3780         unsigned long flags;
3781         int npolled;
3782
3783         spin_lock_irqsave(&hr_cq->lock, flags);
3784
3785         /*
3786          * When the device starts to reset, the state is RST_DOWN. At this time,
3787          * there may still be some valid CQEs in the hardware that are not
3788          * polled. Therefore, it is not allowed to switch to the software mode
3789          * immediately. When the state changes to UNINIT, CQE no longer exists
3790          * in the hardware, and then switch to software mode.
3791          */
3792         if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
3793                 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
3794                 goto out;
3795         }
3796
3797         for (npolled = 0; npolled < num_entries; ++npolled) {
3798                 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
3799                         break;
3800         }
3801
3802         if (npolled)
3803                 update_cq_db(hr_dev, hr_cq);
3804
3805 out:
3806         spin_unlock_irqrestore(&hr_cq->lock, flags);
3807
3808         return npolled;
3809 }
3810
3811 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
3812                               int step_idx, u16 *mbox_op)
3813 {
3814         u16 op;
3815
3816         switch (type) {
3817         case HEM_TYPE_QPC:
3818                 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
3819                 break;
3820         case HEM_TYPE_MTPT:
3821                 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
3822                 break;
3823         case HEM_TYPE_CQC:
3824                 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
3825                 break;
3826         case HEM_TYPE_SRQC:
3827                 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
3828                 break;
3829         case HEM_TYPE_SCCC:
3830                 op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
3831                 break;
3832         case HEM_TYPE_QPC_TIMER:
3833                 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
3834                 break;
3835         case HEM_TYPE_CQC_TIMER:
3836                 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
3837                 break;
3838         default:
3839                 dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type);
3840                 return -EINVAL;
3841         }
3842
3843         *mbox_op = op + step_idx;
3844
3845         return 0;
3846 }
3847
3848 static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
3849                                dma_addr_t base_addr)
3850 {
3851         struct hns_roce_cmq_desc desc;
3852         struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
3853         u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);
3854         u64 addr = to_hr_hw_page_addr(base_addr);
3855
3856         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
3857
3858         hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
3859         hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
3860         hr_reg_write(req, CFG_GMV_BT_IDX, idx);
3861
3862         return hns_roce_cmq_send(hr_dev, &desc, 1);
3863 }
3864
3865 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
3866                          dma_addr_t base_addr, u32 hem_type, int step_idx)
3867 {
3868         int ret;
3869         u16 op;
3870
3871         if (unlikely(hem_type == HEM_TYPE_GMV))
3872                 return config_gmv_ba_to_hw(hr_dev, obj, base_addr);
3873
3874         if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx))
3875                 return 0;
3876
3877         ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &op);
3878         if (ret < 0)
3879                 return ret;
3880
3881         return config_hem_ba_to_hw(hr_dev, obj, base_addr, op);
3882 }
3883
3884 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
3885                                struct hns_roce_hem_table *table, int obj,
3886                                int step_idx)
3887 {
3888         struct hns_roce_hem_iter iter;
3889         struct hns_roce_hem_mhop mhop;
3890         struct hns_roce_hem *hem;
3891         unsigned long mhop_obj = obj;
3892         int i, j, k;
3893         int ret = 0;
3894         u64 hem_idx = 0;
3895         u64 l1_idx = 0;
3896         u64 bt_ba = 0;
3897         u32 chunk_ba_num;
3898         u32 hop_num;
3899
3900         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3901                 return 0;
3902
3903         hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
3904         i = mhop.l0_idx;
3905         j = mhop.l1_idx;
3906         k = mhop.l2_idx;
3907         hop_num = mhop.hop_num;
3908         chunk_ba_num = mhop.bt_chunk_size / 8;
3909
3910         if (hop_num == 2) {
3911                 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
3912                           k;
3913                 l1_idx = i * chunk_ba_num + j;
3914         } else if (hop_num == 1) {
3915                 hem_idx = i * chunk_ba_num + j;
3916         } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
3917                 hem_idx = i;
3918         }
3919
3920         if (table->type == HEM_TYPE_SCCC)
3921                 obj = mhop.l0_idx;
3922
3923         if (check_whether_last_step(hop_num, step_idx)) {
3924                 hem = table->hem[hem_idx];
3925                 for (hns_roce_hem_first(hem, &iter);
3926                      !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
3927                         bt_ba = hns_roce_hem_addr(&iter);
3928                         ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
3929                                             step_idx);
3930                 }
3931         } else {
3932                 if (step_idx == 0)
3933                         bt_ba = table->bt_l0_dma_addr[i];
3934                 else if (step_idx == 1 && hop_num == 2)
3935                         bt_ba = table->bt_l1_dma_addr[l1_idx];
3936
3937                 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
3938         }
3939
3940         return ret;
3941 }
3942
3943 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3944                                  struct hns_roce_hem_table *table, int obj,
3945                                  int step_idx)
3946 {
3947         struct device *dev = hr_dev->dev;
3948         struct hns_roce_cmd_mailbox *mailbox;
3949         int ret;
3950         u16 op = 0xff;
3951
3952         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3953                 return 0;
3954
3955         switch (table->type) {
3956         case HEM_TYPE_QPC:
3957                 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3958                 break;
3959         case HEM_TYPE_MTPT:
3960                 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3961                 break;
3962         case HEM_TYPE_CQC:
3963                 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3964                 break;
3965         case HEM_TYPE_SRQC:
3966                 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3967                 break;
3968         case HEM_TYPE_SCCC:
3969         case HEM_TYPE_QPC_TIMER:
3970         case HEM_TYPE_CQC_TIMER:
3971         case HEM_TYPE_GMV:
3972                 return 0;
3973         default:
3974                 dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
3975                          table->type);
3976                 return 0;
3977         }
3978
3979         op += step_idx;
3980
3981         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3982         if (IS_ERR(mailbox))
3983                 return PTR_ERR(mailbox);
3984
3985         /* configure the tag and op */
3986         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3987                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
3988
3989         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3990         return ret;
3991 }
3992
3993 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
3994                                  struct hns_roce_v2_qp_context *context,
3995                                  struct hns_roce_v2_qp_context *qpc_mask,
3996                                  struct hns_roce_qp *hr_qp)
3997 {
3998         struct hns_roce_cmd_mailbox *mailbox;
3999         int qpc_size;
4000         int ret;
4001
4002         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4003         if (IS_ERR(mailbox))
4004                 return PTR_ERR(mailbox);
4005
4006         /* The qpc size of HIP08 is only 256B, which is half of HIP09 */
4007         qpc_size = hr_dev->caps.qpc_sz;
4008         memcpy(mailbox->buf, context, qpc_size);
4009         memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
4010
4011         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
4012                                 HNS_ROCE_CMD_MODIFY_QPC,
4013                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
4014
4015         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4016
4017         return ret;
4018 }
4019
4020 static void set_access_flags(struct hns_roce_qp *hr_qp,
4021                              struct hns_roce_v2_qp_context *context,
4022                              struct hns_roce_v2_qp_context *qpc_mask,
4023                              const struct ib_qp_attr *attr, int attr_mask)
4024 {
4025         u8 dest_rd_atomic;
4026         u32 access_flags;
4027
4028         dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
4029                          attr->max_dest_rd_atomic : hr_qp->resp_depth;
4030
4031         access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
4032                        attr->qp_access_flags : hr_qp->atomic_rd_en;
4033
4034         if (!dest_rd_atomic)
4035                 access_flags &= IB_ACCESS_REMOTE_WRITE;
4036
4037         hr_reg_write_bool(context, QPC_RRE,
4038                           access_flags & IB_ACCESS_REMOTE_READ);
4039         hr_reg_clear(qpc_mask, QPC_RRE);
4040
4041         hr_reg_write_bool(context, QPC_RWE,
4042                           access_flags & IB_ACCESS_REMOTE_WRITE);
4043         hr_reg_clear(qpc_mask, QPC_RWE);
4044
4045         hr_reg_write_bool(context, QPC_ATE,
4046                           access_flags & IB_ACCESS_REMOTE_ATOMIC);
4047         hr_reg_clear(qpc_mask, QPC_ATE);
4048         hr_reg_write_bool(context, QPC_EXT_ATE,
4049                           access_flags & IB_ACCESS_REMOTE_ATOMIC);
4050         hr_reg_clear(qpc_mask, QPC_EXT_ATE);
4051 }
4052
4053 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
4054                             struct hns_roce_v2_qp_context *context,
4055                             struct hns_roce_v2_qp_context *qpc_mask)
4056 {
4057         hr_reg_write(context, QPC_SGE_SHIFT,
4058                      to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
4059                                              hr_qp->sge.sge_shift));
4060
4061         hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp->sq.wqe_cnt));
4062
4063         hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp->rq.wqe_cnt));
4064 }
4065
4066 static inline int get_cqn(struct ib_cq *ib_cq)
4067 {
4068         return ib_cq ? to_hr_cq(ib_cq)->cqn : 0;
4069 }
4070
4071 static inline int get_pdn(struct ib_pd *ib_pd)
4072 {
4073         return ib_pd ? to_hr_pd(ib_pd)->pdn : 0;
4074 }
4075
4076 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
4077                                     const struct ib_qp_attr *attr,
4078                                     int attr_mask,
4079                                     struct hns_roce_v2_qp_context *context,
4080                                     struct hns_roce_v2_qp_context *qpc_mask)
4081 {
4082         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4083         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4084
4085         /*
4086          * In v2 engine, software pass context and context mask to hardware
4087          * when modifying qp. If software need modify some fields in context,
4088          * we should set all bits of the relevant fields in context mask to
4089          * 0 at the same time, else set them to 0x1.
4090          */
4091         hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));
4092
4093         hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));
4094
4095         hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs));
4096
4097         set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
4098
4099         /* No VLAN need to set 0xFFF */
4100         hr_reg_write(context, QPC_VLAN_ID, 0xfff);
4101
4102         if (ibqp->qp_type == IB_QPT_XRC_TGT) {
4103                 context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn);
4104
4105                 hr_reg_enable(context, QPC_XRC_QP_TYPE);
4106         }
4107
4108         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
4109                 hr_reg_enable(context, QPC_RQ_RECORD_EN);
4110
4111         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
4112                 hr_reg_enable(context, QPC_OWNER_MODE);
4113
4114         hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L,
4115                      lower_32_bits(hr_qp->rdb.dma) >> 1);
4116         hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,
4117                      upper_32_bits(hr_qp->rdb.dma));
4118
4119         if (ibqp->qp_type != IB_QPT_UD && ibqp->qp_type != IB_QPT_GSI)
4120                 hr_reg_write_bool(context, QPC_RQIE,
4121                              hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE);
4122
4123         hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
4124
4125         if (ibqp->srq) {
4126                 hr_reg_enable(context, QPC_SRQ_EN);
4127                 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);
4128         }
4129
4130         hr_reg_enable(context, QPC_FRE);
4131
4132         hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));
4133
4134         if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ)
4135                 return;
4136
4137         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
4138                 hr_reg_enable(&context->ext, QPCEX_STASH);
4139 }
4140
4141 static void modify_qp_init_to_init(struct ib_qp *ibqp,
4142                                    const struct ib_qp_attr *attr, int attr_mask,
4143                                    struct hns_roce_v2_qp_context *context,
4144                                    struct hns_roce_v2_qp_context *qpc_mask)
4145 {
4146         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4147
4148         /*
4149          * In v2 engine, software pass context and context mask to hardware
4150          * when modifying qp. If software need modify some fields in context,
4151          * we should set all bits of the relevant fields in context mask to
4152          * 0 at the same time, else set them to 0x1.
4153          */
4154         hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));
4155         hr_reg_clear(qpc_mask, QPC_TST);
4156
4157         hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));
4158         hr_reg_clear(qpc_mask, QPC_PD);
4159
4160         hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
4161         hr_reg_clear(qpc_mask, QPC_RX_CQN);
4162
4163         hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));
4164         hr_reg_clear(qpc_mask, QPC_TX_CQN);
4165
4166         if (ibqp->srq) {
4167                 hr_reg_enable(context, QPC_SRQ_EN);
4168                 hr_reg_clear(qpc_mask, QPC_SRQ_EN);
4169                 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);
4170                 hr_reg_clear(qpc_mask, QPC_SRQN);
4171         }
4172
4173         if (attr_mask & IB_QP_DEST_QPN) {
4174                 hr_reg_write(context, QPC_DQPN, hr_qp->qpn);
4175                 hr_reg_clear(qpc_mask, QPC_DQPN);
4176         }
4177 }
4178
4179 static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
4180                             struct hns_roce_qp *hr_qp,
4181                             struct hns_roce_v2_qp_context *context,
4182                             struct hns_roce_v2_qp_context *qpc_mask)
4183 {
4184         u64 mtts[MTT_MIN_COUNT] = { 0 };
4185         u64 wqe_sge_ba;
4186         int count;
4187
4188         /* Search qp buf's mtts */
4189         count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
4190                                   MTT_MIN_COUNT, &wqe_sge_ba);
4191         if (hr_qp->rq.wqe_cnt && count < 1) {
4192                 ibdev_err(&hr_dev->ib_dev,
4193                           "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
4194                 return -EINVAL;
4195         }
4196
4197         context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
4198         qpc_mask->wqe_sge_ba = 0;
4199
4200         /*
4201          * In v2 engine, software pass context and context mask to hardware
4202          * when modifying qp. If software need modify some fields in context,
4203          * we should set all bits of the relevant fields in context mask to
4204          * 0 at the same time, else set them to 0x1.
4205          */
4206         hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> (32 + 3));
4207         hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H);
4208
4209         hr_reg_write(context, QPC_SQ_HOP_NUM,
4210                      to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
4211                                       hr_qp->sq.wqe_cnt));
4212         hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM);
4213
4214         hr_reg_write(context, QPC_SGE_HOP_NUM,
4215                      to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
4216                                       hr_qp->sge.sge_cnt));
4217         hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM);
4218
4219         hr_reg_write(context, QPC_RQ_HOP_NUM,
4220                      to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
4221                                       hr_qp->rq.wqe_cnt));
4222
4223         hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM);
4224
4225         hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ,
4226                      to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
4227         hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ);
4228
4229         hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ,
4230                      to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
4231         hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ);
4232
4233         context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
4234         qpc_mask->rq_cur_blk_addr = 0;
4235
4236         hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H,
4237                      upper_32_bits(to_hr_hw_page_addr(mtts[0])));
4238         hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H);
4239
4240         context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
4241         qpc_mask->rq_nxt_blk_addr = 0;
4242
4243         hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H,
4244                      upper_32_bits(to_hr_hw_page_addr(mtts[1])));
4245         hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H);
4246
4247         return 0;
4248 }
4249
4250 static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
4251                             struct hns_roce_qp *hr_qp,
4252                             struct hns_roce_v2_qp_context *context,
4253                             struct hns_roce_v2_qp_context *qpc_mask)
4254 {
4255         struct ib_device *ibdev = &hr_dev->ib_dev;
4256         u64 sge_cur_blk = 0;
4257         u64 sq_cur_blk = 0;
4258         int count;
4259
4260         /* search qp buf's mtts */
4261         count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
4262         if (count < 1) {
4263                 ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n",
4264                           hr_qp->qpn);
4265                 return -EINVAL;
4266         }
4267         if (hr_qp->sge.sge_cnt > 0) {
4268                 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
4269                                           hr_qp->sge.offset,
4270                                           &sge_cur_blk, 1, NULL);
4271                 if (count < 1) {
4272                         ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
4273                                   hr_qp->qpn);
4274                         return -EINVAL;
4275                 }
4276         }
4277
4278         /*
4279          * In v2 engine, software pass context and context mask to hardware
4280          * when modifying qp. If software need modify some fields in context,
4281          * we should set all bits of the relevant fields in context mask to
4282          * 0 at the same time, else set them to 0x1.
4283          */
4284         hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L,
4285                      lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4286         hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H,
4287                      upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4288         hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L);
4289         hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H);
4290
4291         hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L,
4292                      lower_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4293         hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H,
4294                      upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4295         hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L);
4296         hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H);
4297
4298         hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L,
4299                      lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4300         hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H,
4301                      upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4302         hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L);
4303         hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H);
4304
4305         return 0;
4306 }
4307
4308 static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,
4309                                   const struct ib_qp_attr *attr)
4310 {
4311         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
4312                 return IB_MTU_4096;
4313
4314         return attr->path_mtu;
4315 }
4316
4317 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
4318                                  const struct ib_qp_attr *attr, int attr_mask,
4319                                  struct hns_roce_v2_qp_context *context,
4320                                  struct hns_roce_v2_qp_context *qpc_mask)
4321 {
4322         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4323         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4324         struct ib_device *ibdev = &hr_dev->ib_dev;
4325         dma_addr_t trrl_ba;
4326         dma_addr_t irrl_ba;
4327         enum ib_mtu ib_mtu;
4328         u8 lp_pktn_ini;
4329         u64 *mtts;
4330         u8 *dmac;
4331         u8 *smac;
4332         u32 port;
4333         int mtu;
4334         int ret;
4335
4336         ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
4337         if (ret) {
4338                 ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
4339                 return ret;
4340         }
4341
4342         /* Search IRRL's mtts */
4343         mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
4344                                    hr_qp->qpn, &irrl_ba);
4345         if (!mtts) {
4346                 ibdev_err(ibdev, "failed to find qp irrl_table.\n");
4347                 return -EINVAL;
4348         }
4349
4350         /* Search TRRL's mtts */
4351         mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
4352                                    hr_qp->qpn, &trrl_ba);
4353         if (!mtts) {
4354                 ibdev_err(ibdev, "failed to find qp trrl_table.\n");
4355                 return -EINVAL;
4356         }
4357
4358         if (attr_mask & IB_QP_ALT_PATH) {
4359                 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
4360                           attr_mask);
4361                 return -EINVAL;
4362         }
4363
4364         hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> 4);
4365         hr_reg_clear(qpc_mask, QPC_TRRL_BA_L);
4366         context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4));
4367         qpc_mask->trrl_ba = 0;
4368         hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> (32 + 16 + 4));
4369         hr_reg_clear(qpc_mask, QPC_TRRL_BA_H);
4370
4371         context->irrl_ba = cpu_to_le32(irrl_ba >> 6);
4372         qpc_mask->irrl_ba = 0;
4373         hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> (32 + 6));
4374         hr_reg_clear(qpc_mask, QPC_IRRL_BA_H);
4375
4376         hr_reg_enable(context, QPC_RMT_E2E);
4377         hr_reg_clear(qpc_mask, QPC_RMT_E2E);
4378
4379         hr_reg_write(context, QPC_SIG_TYPE, hr_qp->sq_signal_bits);
4380         hr_reg_clear(qpc_mask, QPC_SIG_TYPE);
4381
4382         port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
4383
4384         smac = (u8 *)hr_dev->dev_addr[port];
4385         dmac = (u8 *)attr->ah_attr.roce.dmac;
4386         /* when dmac equals smac or loop_idc is 1, it should loopback */
4387         if (ether_addr_equal_unaligned(dmac, smac) ||
4388             hr_dev->loop_idc == 0x1) {
4389                 hr_reg_write(context, QPC_LBI, hr_dev->loop_idc);
4390                 hr_reg_clear(qpc_mask, QPC_LBI);
4391         }
4392
4393         if (attr_mask & IB_QP_DEST_QPN) {
4394                 hr_reg_write(context, QPC_DQPN, attr->dest_qp_num);
4395                 hr_reg_clear(qpc_mask, QPC_DQPN);
4396         }
4397
4398         memcpy(&(context->dmac), dmac, sizeof(u32));
4399         hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4])));
4400         qpc_mask->dmac = 0;
4401         hr_reg_clear(qpc_mask, QPC_DMAC_H);
4402
4403         ib_mtu = get_mtu(ibqp, attr);
4404         hr_qp->path_mtu = ib_mtu;
4405
4406         mtu = ib_mtu_enum_to_int(ib_mtu);
4407         if (WARN_ON(mtu < 0))
4408                 return -EINVAL;
4409
4410         if (attr_mask & IB_QP_PATH_MTU) {
4411                 hr_reg_write(context, QPC_MTU, ib_mtu);
4412                 hr_reg_clear(qpc_mask, QPC_MTU);
4413         }
4414
4415 #define MAX_LP_MSG_LEN 65536
4416         /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */
4417         lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / mtu);
4418
4419         hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini);
4420         hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI);
4421
4422         /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */
4423         hr_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini);
4424         hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ);
4425
4426         hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR);
4427         hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN);
4428         hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE);
4429
4430         context->rq_rnr_timer = 0;
4431         qpc_mask->rq_rnr_timer = 0;
4432
4433         hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX);
4434         hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX);
4435
4436         /* rocee send 2^lp_sgen_ini segs every time */
4437         hr_reg_write(context, QPC_LP_SGEN_INI, 3);
4438         hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);
4439
4440         return 0;
4441 }
4442
4443 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
4444                                 const struct ib_qp_attr *attr, int attr_mask,
4445                                 struct hns_roce_v2_qp_context *context,
4446                                 struct hns_roce_v2_qp_context *qpc_mask)
4447 {
4448         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4449         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4450         struct ib_device *ibdev = &hr_dev->ib_dev;
4451         int ret;
4452
4453         /* Not support alternate path and path migration */
4454         if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {
4455                 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
4456                 return -EINVAL;
4457         }
4458
4459         ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
4460         if (ret) {
4461                 ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
4462                 return ret;
4463         }
4464
4465         /*
4466          * Set some fields in context to zero, Because the default values
4467          * of all fields in context are zero, we need not set them to 0 again.
4468          * but we should set the relevant fields of context mask to 0.
4469          */
4470         hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX);
4471
4472         hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN);
4473
4474         hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE);
4475         hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD);
4476         hr_reg_clear(qpc_mask, QPC_IRRL_PSN);
4477
4478         hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL);
4479
4480         hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN);
4481
4482         hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG);
4483
4484         hr_reg_clear(qpc_mask, QPC_CHECK_FLG);
4485
4486         hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD);
4487
4488         return 0;
4489 }
4490
4491 static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4492 {
4493         if (!fl)
4494                 fl = rdma_calc_flow_label(lqpn, rqpn);
4495
4496         return rdma_flow_label_to_udp_sport(fl);
4497 }
4498
4499 static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4500                            u32 *dip_idx)
4501 {
4502         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4503         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4504         struct hns_roce_dip *hr_dip;
4505         unsigned long flags;
4506         int ret = 0;
4507
4508         spin_lock_irqsave(&hr_dev->dip_list_lock, flags);
4509
4510         list_for_each_entry(hr_dip, &hr_dev->dip_list, node) {
4511                 if (!memcmp(grh->dgid.raw, hr_dip->dgid, 16))
4512                         goto out;
4513         }
4514
4515         /* If no dgid is found, a new dip and a mapping between dgid and
4516          * dip_idx will be created.
4517          */
4518         hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC);
4519         if (!hr_dip) {
4520                 ret = -ENOMEM;
4521                 goto out;
4522         }
4523
4524         memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4525         hr_dip->dip_idx = *dip_idx = ibqp->qp_num;
4526         list_add_tail(&hr_dip->node, &hr_dev->dip_list);
4527
4528 out:
4529         spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);
4530         return ret;
4531 }
4532
4533 enum {
4534         CONG_DCQCN,
4535         CONG_WINDOW,
4536 };
4537
4538 enum {
4539         UNSUPPORT_CONG_LEVEL,
4540         SUPPORT_CONG_LEVEL,
4541 };
4542
4543 enum {
4544         CONG_LDCP,
4545         CONG_HC3,
4546 };
4547
4548 enum {
4549         DIP_INVALID,
4550         DIP_VALID,
4551 };
4552
4553 enum {
4554         WND_LIMIT,
4555         WND_UNLIMIT,
4556 };
4557
4558 static int check_cong_type(struct ib_qp *ibqp,
4559                            struct hns_roce_congestion_algorithm *cong_alg)
4560 {
4561         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4562
4563         /* different congestion types match different configurations */
4564         switch (hr_dev->caps.cong_type) {
4565         case CONG_TYPE_DCQCN:
4566                 cong_alg->alg_sel = CONG_DCQCN;
4567                 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4568                 cong_alg->dip_vld = DIP_INVALID;
4569                 cong_alg->wnd_mode_sel = WND_LIMIT;
4570                 break;
4571         case CONG_TYPE_LDCP:
4572                 cong_alg->alg_sel = CONG_WINDOW;
4573                 cong_alg->alg_sub_sel = CONG_LDCP;
4574                 cong_alg->dip_vld = DIP_INVALID;
4575                 cong_alg->wnd_mode_sel = WND_UNLIMIT;
4576                 break;
4577         case CONG_TYPE_HC3:
4578                 cong_alg->alg_sel = CONG_WINDOW;
4579                 cong_alg->alg_sub_sel = CONG_HC3;
4580                 cong_alg->dip_vld = DIP_INVALID;
4581                 cong_alg->wnd_mode_sel = WND_LIMIT;
4582                 break;
4583         case CONG_TYPE_DIP:
4584                 cong_alg->alg_sel = CONG_DCQCN;
4585                 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4586                 cong_alg->dip_vld = DIP_VALID;
4587                 cong_alg->wnd_mode_sel = WND_LIMIT;
4588                 break;
4589         default:
4590                 ibdev_err(&hr_dev->ib_dev,
4591                           "error type(%u) for congestion selection.\n",
4592                           hr_dev->caps.cong_type);
4593                 return -EINVAL;
4594         }
4595
4596         return 0;
4597 }
4598
4599 static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4600                            struct hns_roce_v2_qp_context *context,
4601                            struct hns_roce_v2_qp_context *qpc_mask)
4602 {
4603         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4604         struct hns_roce_congestion_algorithm cong_field;
4605         struct ib_device *ibdev = ibqp->device;
4606         struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
4607         u32 dip_idx = 0;
4608         int ret;
4609
4610         if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ||
4611             grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE)
4612                 return 0;
4613
4614         ret = check_cong_type(ibqp, &cong_field);
4615         if (ret)
4616                 return ret;
4617
4618         hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id +
4619                      hr_dev->caps.cong_type * HNS_ROCE_CONG_SIZE);
4620         hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID);
4621         hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel);
4622         hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SEL);
4623         hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL,
4624                      cong_field.alg_sub_sel);
4625         hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL);
4626         hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld);
4627         hr_reg_clear(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD);
4628         hr_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN,
4629                      cong_field.wnd_mode_sel);
4630         hr_reg_clear(&qpc_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN);
4631
4632         /* if dip is disabled, there is no need to set dip idx */
4633         if (cong_field.dip_vld == 0)
4634                 return 0;
4635
4636         ret = get_dip_ctx_idx(ibqp, attr, &dip_idx);
4637         if (ret) {
4638                 ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret);
4639                 return ret;
4640         }
4641
4642         hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx);
4643         hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0);
4644
4645         return 0;
4646 }
4647
4648 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
4649                                 const struct ib_qp_attr *attr,
4650                                 int attr_mask,
4651                                 struct hns_roce_v2_qp_context *context,
4652                                 struct hns_roce_v2_qp_context *qpc_mask)
4653 {
4654         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4655         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4656         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4657         struct ib_device *ibdev = &hr_dev->ib_dev;
4658         const struct ib_gid_attr *gid_attr = NULL;
4659         int is_roce_protocol;
4660         u16 vlan_id = 0xffff;
4661         bool is_udp = false;
4662         u8 ib_port;
4663         u8 hr_port;
4664         int ret;
4665
4666         ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
4667         hr_port = ib_port - 1;
4668         is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4669                            rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4670
4671         if (is_roce_protocol) {
4672                 gid_attr = attr->ah_attr.grh.sgid_attr;
4673                 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
4674                 if (ret)
4675                         return ret;
4676
4677                 if (gid_attr)
4678                         is_udp = (gid_attr->gid_type ==
4679                                  IB_GID_TYPE_ROCE_UDP_ENCAP);
4680         }
4681
4682         /* Only HIP08 needs to set the vlan_en bits in QPC */
4683         if (vlan_id < VLAN_N_VID &&
4684             hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
4685                 hr_reg_enable(context, QPC_RQ_VLAN_EN);
4686                 hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN);
4687                 hr_reg_enable(context, QPC_SQ_VLAN_EN);
4688                 hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN);
4689         }
4690
4691         hr_reg_write(context, QPC_VLAN_ID, vlan_id);
4692         hr_reg_clear(qpc_mask, QPC_VLAN_ID);
4693
4694         if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4695                 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
4696                           grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
4697                 return -EINVAL;
4698         }
4699
4700         if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4701                 ibdev_err(ibdev, "ah attr is not RDMA roce type\n");
4702                 return -EINVAL;
4703         }
4704
4705         hr_reg_write(context, QPC_UDPSPN,
4706                      is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num,
4707                                             attr->dest_qp_num) : 0);
4708
4709         hr_reg_clear(qpc_mask, QPC_UDPSPN);
4710
4711         hr_reg_write(context, QPC_GMV_IDX, grh->sgid_index);
4712
4713         hr_reg_clear(qpc_mask, QPC_GMV_IDX);
4714
4715         hr_reg_write(context, QPC_HOPLIMIT, grh->hop_limit);
4716         hr_reg_clear(qpc_mask, QPC_HOPLIMIT);
4717
4718         ret = fill_cong_field(ibqp, attr, context, qpc_mask);
4719         if (ret)
4720                 return ret;
4721
4722         hr_reg_write(context, QPC_TC, get_tclass(&attr->ah_attr.grh));
4723         hr_reg_clear(qpc_mask, QPC_TC);
4724
4725         hr_reg_write(context, QPC_FL, grh->flow_label);
4726         hr_reg_clear(qpc_mask, QPC_FL);
4727         memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4728         memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4729
4730         hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4731         if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) {
4732                 ibdev_err(ibdev,
4733                           "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n",
4734                           hr_qp->sl, MAX_SERVICE_LEVEL);
4735                 return -EINVAL;
4736         }
4737
4738         hr_reg_write(context, QPC_SL, hr_qp->sl);
4739         hr_reg_clear(qpc_mask, QPC_SL);
4740
4741         return 0;
4742 }
4743
4744 static bool check_qp_state(enum ib_qp_state cur_state,
4745                            enum ib_qp_state new_state)
4746 {
4747         static const bool sm[][IB_QPS_ERR + 1] = {
4748                 [IB_QPS_RESET] = { [IB_QPS_RESET] = true,
4749                                    [IB_QPS_INIT] = true },
4750                 [IB_QPS_INIT] = { [IB_QPS_RESET] = true,
4751                                   [IB_QPS_INIT] = true,
4752                                   [IB_QPS_RTR] = true,
4753                                   [IB_QPS_ERR] = true },
4754                 [IB_QPS_RTR] = { [IB_QPS_RESET] = true,
4755                                  [IB_QPS_RTS] = true,
4756                                  [IB_QPS_ERR] = true },
4757                 [IB_QPS_RTS] = { [IB_QPS_RESET] = true,
4758                                  [IB_QPS_RTS] = true,
4759                                  [IB_QPS_ERR] = true },
4760                 [IB_QPS_SQD] = {},
4761                 [IB_QPS_SQE] = {},
4762                 [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
4763         };
4764
4765         return sm[cur_state][new_state];
4766 }
4767
4768 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
4769                                       const struct ib_qp_attr *attr,
4770                                       int attr_mask,
4771                                       enum ib_qp_state cur_state,
4772                                       enum ib_qp_state new_state,
4773                                       struct hns_roce_v2_qp_context *context,
4774                                       struct hns_roce_v2_qp_context *qpc_mask)
4775 {
4776         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4777         int ret = 0;
4778
4779         if (!check_qp_state(cur_state, new_state)) {
4780                 ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");
4781                 return -EINVAL;
4782         }
4783
4784         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4785                 memset(qpc_mask, 0, hr_dev->caps.qpc_sz);
4786                 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4787                                         qpc_mask);
4788         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4789                 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4790                                        qpc_mask);
4791         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4792                 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4793                                             qpc_mask);
4794         } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4795                 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4796                                            qpc_mask);
4797         }
4798
4799         return ret;
4800 }
4801
4802 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
4803                                       const struct ib_qp_attr *attr,
4804                                       int attr_mask,
4805                                       struct hns_roce_v2_qp_context *context,
4806                                       struct hns_roce_v2_qp_context *qpc_mask)
4807 {
4808         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4809         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4810         int ret = 0;
4811
4812         if (attr_mask & IB_QP_AV) {
4813                 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
4814                                            qpc_mask);
4815                 if (ret)
4816                         return ret;
4817         }
4818
4819         if (attr_mask & IB_QP_TIMEOUT) {
4820                 if (attr->timeout < 31) {
4821                         hr_reg_write(context, QPC_AT, attr->timeout);
4822                         hr_reg_clear(qpc_mask, QPC_AT);
4823                 } else {
4824                         ibdev_warn(&hr_dev->ib_dev,
4825                                    "Local ACK timeout shall be 0 to 30.\n");
4826                 }
4827         }
4828
4829         if (attr_mask & IB_QP_RETRY_CNT) {
4830                 hr_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt);
4831                 hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT);
4832
4833                 hr_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt);
4834                 hr_reg_clear(qpc_mask, QPC_RETRY_CNT);
4835         }
4836
4837         if (attr_mask & IB_QP_RNR_RETRY) {
4838                 hr_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry);
4839                 hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT);
4840
4841                 hr_reg_write(context, QPC_RNR_CNT, attr->rnr_retry);
4842                 hr_reg_clear(qpc_mask, QPC_RNR_CNT);
4843         }
4844
4845         if (attr_mask & IB_QP_SQ_PSN) {
4846                 hr_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn);
4847                 hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN);
4848
4849                 hr_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn);
4850                 hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN);
4851
4852                 hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->sq_psn);
4853                 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L);
4854
4855                 hr_reg_write(context, QPC_RETRY_MSG_PSN_H,
4856                              attr->sq_psn >> RETRY_MSG_PSN_SHIFT);
4857                 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H);
4858
4859                 hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn);
4860                 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN);
4861
4862                 hr_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn);
4863                 hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN);
4864         }
4865
4866         if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4867              attr->max_dest_rd_atomic) {
4868                 hr_reg_write(context, QPC_RR_MAX,
4869                              fls(attr->max_dest_rd_atomic - 1));
4870                 hr_reg_clear(qpc_mask, QPC_RR_MAX);
4871         }
4872
4873         if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4874                 hr_reg_write(context, QPC_SR_MAX, fls(attr->max_rd_atomic - 1));
4875                 hr_reg_clear(qpc_mask, QPC_SR_MAX);
4876         }
4877
4878         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4879                 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4880
4881         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4882                 hr_reg_write(context, QPC_MIN_RNR_TIME, attr->min_rnr_timer);
4883                 hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME);
4884         }
4885
4886         if (attr_mask & IB_QP_RQ_PSN) {
4887                 hr_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn);
4888                 hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN);
4889
4890                 hr_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1);
4891                 hr_reg_clear(qpc_mask, QPC_RAQ_PSN);
4892         }
4893
4894         if (attr_mask & IB_QP_QKEY) {
4895                 context->qkey_xrcd = cpu_to_le32(attr->qkey);
4896                 qpc_mask->qkey_xrcd = 0;
4897                 hr_qp->qkey = attr->qkey;
4898         }
4899
4900         return ret;
4901 }
4902
4903 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
4904                                           const struct ib_qp_attr *attr,
4905                                           int attr_mask)
4906 {
4907         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4908         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4909
4910         if (attr_mask & IB_QP_ACCESS_FLAGS)
4911                 hr_qp->atomic_rd_en = attr->qp_access_flags;
4912
4913         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4914                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
4915         if (attr_mask & IB_QP_PORT) {
4916                 hr_qp->port = attr->port_num - 1;
4917                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4918         }
4919 }
4920
4921 static void clear_qp(struct hns_roce_qp *hr_qp)
4922 {
4923         struct ib_qp *ibqp = &hr_qp->ibqp;
4924
4925         if (ibqp->send_cq)
4926                 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4927                                      hr_qp->qpn, NULL);
4928
4929         if (ibqp->recv_cq  && ibqp->recv_cq != ibqp->send_cq)
4930                 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq),
4931                                      hr_qp->qpn, ibqp->srq ?
4932                                      to_hr_srq(ibqp->srq) : NULL);
4933
4934         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
4935                 *hr_qp->rdb.db_record = 0;
4936
4937         hr_qp->rq.head = 0;
4938         hr_qp->rq.tail = 0;
4939         hr_qp->sq.head = 0;
4940         hr_qp->sq.tail = 0;
4941         hr_qp->next_sge = 0;
4942 }
4943
4944 static void v2_set_flushed_fields(struct ib_qp *ibqp,
4945                                   struct hns_roce_v2_qp_context *context,
4946                                   struct hns_roce_v2_qp_context *qpc_mask)
4947 {
4948         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4949         unsigned long sq_flag = 0;
4950         unsigned long rq_flag = 0;
4951
4952         if (ibqp->qp_type == IB_QPT_XRC_TGT)
4953                 return;
4954
4955         spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);
4956         hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);
4957         hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);
4958         hr_qp->state = IB_QPS_ERR;
4959         spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
4960
4961         if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */
4962                 return;
4963
4964         spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);
4965         hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head);
4966         hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX);
4967         spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);
4968 }
4969
4970 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
4971                                  const struct ib_qp_attr *attr,
4972                                  int attr_mask, enum ib_qp_state cur_state,
4973                                  enum ib_qp_state new_state)
4974 {
4975         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4976         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4977         struct hns_roce_v2_qp_context ctx[2];
4978         struct hns_roce_v2_qp_context *context = ctx;
4979         struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
4980         struct ib_device *ibdev = &hr_dev->ib_dev;
4981         int ret;
4982
4983         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
4984                 return -EOPNOTSUPP;
4985
4986         /*
4987          * In v2 engine, software pass context and context mask to hardware
4988          * when modifying qp. If software need modify some fields in context,
4989          * we should set all bits of the relevant fields in context mask to
4990          * 0 at the same time, else set them to 0x1.
4991          */
4992         memset(context, 0, hr_dev->caps.qpc_sz);
4993         memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
4994
4995         ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
4996                                          new_state, context, qpc_mask);
4997         if (ret)
4998                 goto out;
4999
5000         /* When QP state is err, SQ and RQ WQE should be flushed */
5001         if (new_state == IB_QPS_ERR)
5002                 v2_set_flushed_fields(ibqp, context, qpc_mask);
5003
5004         /* Configure the optional fields */
5005         ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
5006                                          qpc_mask);
5007         if (ret)
5008                 goto out;
5009
5010         hr_reg_write_bool(context, QPC_INV_CREDIT,
5011                           to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC ||
5012                           ibqp->srq);
5013         hr_reg_clear(qpc_mask, QPC_INV_CREDIT);
5014
5015         /* Every status migrate must change state */
5016         hr_reg_write(context, QPC_QP_ST, new_state);
5017         hr_reg_clear(qpc_mask, QPC_QP_ST);
5018
5019         /* SW pass context to HW */
5020         ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
5021         if (ret) {
5022                 ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);
5023                 goto out;
5024         }
5025
5026         hr_qp->state = new_state;
5027
5028         hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
5029
5030         if (new_state == IB_QPS_RESET && !ibqp->uobject)
5031                 clear_qp(hr_qp);
5032
5033 out:
5034         return ret;
5035 }
5036
5037 static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
5038 {
5039         static const enum ib_qp_state map[] = {
5040                 [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
5041                 [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
5042                 [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
5043                 [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
5044                 [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
5045                 [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
5046                 [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
5047                 [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
5048         };
5049
5050         return (state < ARRAY_SIZE(map)) ? map[state] : -1;
5051 }
5052
5053 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
5054                                  struct hns_roce_qp *hr_qp,
5055                                  struct hns_roce_v2_qp_context *hr_context)
5056 {
5057         struct hns_roce_cmd_mailbox *mailbox;
5058         int ret;
5059
5060         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5061         if (IS_ERR(mailbox))
5062                 return PTR_ERR(mailbox);
5063
5064         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
5065                                 HNS_ROCE_CMD_QUERY_QPC,
5066                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
5067         if (ret)
5068                 goto out;
5069
5070         memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
5071
5072 out:
5073         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5074         return ret;
5075 }
5076
5077 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5078                                 int qp_attr_mask,
5079                                 struct ib_qp_init_attr *qp_init_attr)
5080 {
5081         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5082         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5083         struct hns_roce_v2_qp_context context = {};
5084         struct ib_device *ibdev = &hr_dev->ib_dev;
5085         int tmp_qp_state;
5086         int state;
5087         int ret;
5088
5089         memset(qp_attr, 0, sizeof(*qp_attr));
5090         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5091
5092         mutex_lock(&hr_qp->mutex);
5093
5094         if (hr_qp->state == IB_QPS_RESET) {
5095                 qp_attr->qp_state = IB_QPS_RESET;
5096                 ret = 0;
5097                 goto done;
5098         }
5099
5100         ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
5101         if (ret) {
5102                 ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
5103                 ret = -EINVAL;
5104                 goto out;
5105         }
5106
5107         state = hr_reg_read(&context, QPC_QP_ST);
5108         tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
5109         if (tmp_qp_state == -1) {
5110                 ibdev_err(ibdev, "Illegal ib_qp_state\n");
5111                 ret = -EINVAL;
5112                 goto out;
5113         }
5114         hr_qp->state = (u8)tmp_qp_state;
5115         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
5116         qp_attr->path_mtu = (enum ib_mtu)hr_reg_read(&context, QPC_MTU);
5117         qp_attr->path_mig_state = IB_MIG_ARMED;
5118         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
5119         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
5120                 qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
5121
5122         qp_attr->rq_psn = hr_reg_read(&context, QPC_RX_REQ_EPSN);
5123         qp_attr->sq_psn = (u32)hr_reg_read(&context, QPC_SQ_CUR_PSN);
5124         qp_attr->dest_qp_num = (u8)hr_reg_read(&context, QPC_DQPN);
5125         qp_attr->qp_access_flags =
5126                 ((hr_reg_read(&context, QPC_RRE)) << V2_QP_RRE_S) |
5127                 ((hr_reg_read(&context, QPC_RWE)) << V2_QP_RWE_S) |
5128                 ((hr_reg_read(&context, QPC_ATE)) << V2_QP_ATE_S);
5129
5130         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
5131             hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
5132             hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) {
5133                 struct ib_global_route *grh =
5134                         rdma_ah_retrieve_grh(&qp_attr->ah_attr);
5135
5136                 rdma_ah_set_sl(&qp_attr->ah_attr,
5137                                hr_reg_read(&context, QPC_SL));
5138                 grh->flow_label = hr_reg_read(&context, QPC_FL);
5139                 grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);
5140                 grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
5141                 grh->traffic_class = hr_reg_read(&context, QPC_TC);
5142
5143                 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
5144         }
5145
5146         qp_attr->port_num = hr_qp->port + 1;
5147         qp_attr->sq_draining = 0;
5148         qp_attr->max_rd_atomic = 1 << hr_reg_read(&context, QPC_SR_MAX);
5149         qp_attr->max_dest_rd_atomic = 1 << hr_reg_read(&context, QPC_RR_MAX);
5150
5151         qp_attr->min_rnr_timer = (u8)hr_reg_read(&context, QPC_MIN_RNR_TIME);
5152         qp_attr->timeout = (u8)hr_reg_read(&context, QPC_AT);
5153         qp_attr->retry_cnt = hr_reg_read(&context, QPC_RETRY_NUM_INIT);
5154         qp_attr->rnr_retry = hr_reg_read(&context, QPC_RNR_NUM_INIT);
5155
5156 done:
5157         qp_attr->cur_qp_state = qp_attr->qp_state;
5158         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
5159         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
5160         qp_attr->cap.max_inline_data = hr_qp->max_inline_data;
5161
5162         qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
5163         qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
5164
5165         qp_init_attr->qp_context = ibqp->qp_context;
5166         qp_init_attr->qp_type = ibqp->qp_type;
5167         qp_init_attr->recv_cq = ibqp->recv_cq;
5168         qp_init_attr->send_cq = ibqp->send_cq;
5169         qp_init_attr->srq = ibqp->srq;
5170         qp_init_attr->cap = qp_attr->cap;
5171         qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits;
5172
5173 out:
5174         mutex_unlock(&hr_qp->mutex);
5175         return ret;
5176 }
5177
5178 static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp)
5179 {
5180         return ((hr_qp->ibqp.qp_type == IB_QPT_RC ||
5181                  hr_qp->ibqp.qp_type == IB_QPT_UD ||
5182                  hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
5183                  hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&
5184                 hr_qp->state != IB_QPS_RESET);
5185 }
5186
5187 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
5188                                          struct hns_roce_qp *hr_qp,
5189                                          struct ib_udata *udata)
5190 {
5191         struct ib_device *ibdev = &hr_dev->ib_dev;
5192         struct hns_roce_cq *send_cq, *recv_cq;
5193         unsigned long flags;
5194         int ret = 0;
5195
5196         if (modify_qp_is_ok(hr_qp)) {
5197                 /* Modify qp to reset before destroying qp */
5198                 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
5199                                             hr_qp->state, IB_QPS_RESET);
5200                 if (ret)
5201                         ibdev_err(ibdev,
5202                                   "failed to modify QP to RST, ret = %d.\n",
5203                                   ret);
5204         }
5205
5206         send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
5207         recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
5208
5209         spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
5210         hns_roce_lock_cqs(send_cq, recv_cq);
5211
5212         if (!udata) {
5213                 if (recv_cq)
5214                         __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
5215                                                (hr_qp->ibqp.srq ?
5216                                                 to_hr_srq(hr_qp->ibqp.srq) :
5217                                                 NULL));
5218
5219                 if (send_cq && send_cq != recv_cq)
5220                         __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
5221
5222         }
5223
5224         hns_roce_qp_remove(hr_dev, hr_qp);
5225
5226         hns_roce_unlock_cqs(send_cq, recv_cq);
5227         spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
5228
5229         return ret;
5230 }
5231
5232 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
5233 {
5234         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5235         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5236         int ret;
5237
5238         ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
5239         if (ret)
5240                 ibdev_err(&hr_dev->ib_dev,
5241                           "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
5242                           hr_qp->qpn, ret);
5243
5244         hns_roce_qp_destroy(hr_dev, hr_qp, udata);
5245
5246         return 0;
5247 }
5248
5249 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
5250                                             struct hns_roce_qp *hr_qp)
5251 {
5252         struct ib_device *ibdev = &hr_dev->ib_dev;
5253         struct hns_roce_sccc_clr_done *resp;
5254         struct hns_roce_sccc_clr *clr;
5255         struct hns_roce_cmq_desc desc;
5256         int ret, i;
5257
5258         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
5259                 return 0;
5260
5261         mutex_lock(&hr_dev->qp_table.scc_mutex);
5262
5263         /* set scc ctx clear done flag */
5264         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
5265         ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
5266         if (ret) {
5267                 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
5268                 goto out;
5269         }
5270
5271         /* clear scc context */
5272         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
5273         clr = (struct hns_roce_sccc_clr *)desc.data;
5274         clr->qpn = cpu_to_le32(hr_qp->qpn);
5275         ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
5276         if (ret) {
5277                 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
5278                 goto out;
5279         }
5280
5281         /* query scc context clear is done or not */
5282         resp = (struct hns_roce_sccc_clr_done *)desc.data;
5283         for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
5284                 hns_roce_cmq_setup_basic_desc(&desc,
5285                                               HNS_ROCE_OPC_QUERY_SCCC, true);
5286                 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5287                 if (ret) {
5288                         ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",
5289                                   ret);
5290                         goto out;
5291                 }
5292
5293                 if (resp->clr_done)
5294                         goto out;
5295
5296                 msleep(20);
5297         }
5298
5299         ibdev_err(ibdev, "Query SCC clr done flag overtime.\n");
5300         ret = -ETIMEDOUT;
5301
5302 out:
5303         mutex_unlock(&hr_dev->qp_table.scc_mutex);
5304         return ret;
5305 }
5306
5307 #define DMA_IDX_SHIFT 3
5308 #define DMA_WQE_SHIFT 3
5309
5310 static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,
5311                                               struct hns_roce_srq_context *ctx)
5312 {
5313         struct hns_roce_idx_que *idx_que = &srq->idx_que;
5314         struct ib_device *ibdev = srq->ibsrq.device;
5315         struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5316         u64 mtts_idx[MTT_MIN_COUNT] = {};
5317         dma_addr_t dma_handle_idx = 0;
5318         int ret;
5319
5320         /* Get physical address of idx que buf */
5321         ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx,
5322                                 ARRAY_SIZE(mtts_idx), &dma_handle_idx);
5323         if (ret < 1) {
5324                 ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",
5325                           ret);
5326                 return -ENOBUFS;
5327         }
5328
5329         hr_reg_write(ctx, SRQC_IDX_HOP_NUM,
5330                      to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt));
5331
5332         hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT);
5333         hr_reg_write(ctx, SRQC_IDX_BT_BA_H,
5334                      upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT));
5335
5336         hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ,
5337                      to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift));
5338         hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,
5339                      to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift));
5340
5341         hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,
5342                      to_hr_hw_page_addr(mtts_idx[0]));
5343         hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,
5344                      upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
5345
5346         hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,
5347                      to_hr_hw_page_addr(mtts_idx[1]));
5348         hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,
5349                      upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
5350
5351         return 0;
5352 }
5353
5354 static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)
5355 {
5356         struct ib_device *ibdev = srq->ibsrq.device;
5357         struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5358         struct hns_roce_srq_context *ctx = mb_buf;
5359         u64 mtts_wqe[MTT_MIN_COUNT] = {};
5360         dma_addr_t dma_handle_wqe = 0;
5361         int ret;
5362
5363         memset(ctx, 0, sizeof(*ctx));
5364
5365         /* Get the physical address of srq buf */
5366         ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
5367                                 ARRAY_SIZE(mtts_wqe), &dma_handle_wqe);
5368         if (ret < 1) {
5369                 ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",
5370                           ret);
5371                 return -ENOBUFS;
5372         }
5373
5374         hr_reg_write(ctx, SRQC_SRQ_ST, 1);
5375         hr_reg_write_bool(ctx, SRQC_SRQ_TYPE,
5376                           srq->ibsrq.srq_type == IB_SRQT_XRC);
5377         hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn);
5378         hr_reg_write(ctx, SRQC_SRQN, srq->srqn);
5379         hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn);
5380         hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn);
5381         hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt));
5382         hr_reg_write(ctx, SRQC_RQWS,
5383                      srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1));
5384
5385         hr_reg_write(ctx, SRQC_WQE_HOP_NUM,
5386                      to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
5387                                       srq->wqe_cnt));
5388
5389         hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT);
5390         hr_reg_write(ctx, SRQC_WQE_BT_BA_H,
5391                      upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT));
5392
5393         hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ,
5394                      to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
5395         hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,
5396                      to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
5397
5398         return hns_roce_v2_write_srqc_index_queue(srq, ctx);
5399 }
5400
5401 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5402                                   struct ib_srq_attr *srq_attr,
5403                                   enum ib_srq_attr_mask srq_attr_mask,
5404                                   struct ib_udata *udata)
5405 {
5406         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5407         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5408         struct hns_roce_srq_context *srq_context;
5409         struct hns_roce_srq_context *srqc_mask;
5410         struct hns_roce_cmd_mailbox *mailbox;
5411         int ret;
5412
5413         /* Resizing SRQs is not supported yet */
5414         if (srq_attr_mask & IB_SRQ_MAX_WR)
5415                 return -EINVAL;
5416
5417         if (srq_attr_mask & IB_SRQ_LIMIT) {
5418                 if (srq_attr->srq_limit > srq->wqe_cnt)
5419                         return -EINVAL;
5420
5421                 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5422                 if (IS_ERR(mailbox))
5423                         return PTR_ERR(mailbox);
5424
5425                 srq_context = mailbox->buf;
5426                 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
5427
5428                 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
5429
5430                 hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit);
5431                 hr_reg_clear(srqc_mask, SRQC_LIMIT_WL);
5432
5433                 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
5434                                         HNS_ROCE_CMD_MODIFY_SRQC,
5435                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5436                 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5437                 if (ret) {
5438                         ibdev_err(&hr_dev->ib_dev,
5439                                   "failed to handle cmd of modifying SRQ, ret = %d.\n",
5440                                   ret);
5441                         return ret;
5442                 }
5443         }
5444
5445         return 0;
5446 }
5447
5448 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
5449 {
5450         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5451         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5452         struct hns_roce_srq_context *srq_context;
5453         struct hns_roce_cmd_mailbox *mailbox;
5454         int ret;
5455
5456         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5457         if (IS_ERR(mailbox))
5458                 return PTR_ERR(mailbox);
5459
5460         srq_context = mailbox->buf;
5461         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
5462                                 HNS_ROCE_CMD_QUERY_SRQC,
5463                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
5464         if (ret) {
5465                 ibdev_err(&hr_dev->ib_dev,
5466                           "failed to process cmd of querying SRQ, ret = %d.\n",
5467                           ret);
5468                 goto out;
5469         }
5470
5471         attr->srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL);
5472         attr->max_wr = srq->wqe_cnt;
5473         attr->max_sge = srq->max_gs - srq->rsv_sge;
5474
5475 out:
5476         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5477         return ret;
5478 }
5479
5480 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
5481 {
5482         struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
5483         struct hns_roce_v2_cq_context *cq_context;
5484         struct hns_roce_cq *hr_cq = to_hr_cq(cq);
5485         struct hns_roce_v2_cq_context *cqc_mask;
5486         struct hns_roce_cmd_mailbox *mailbox;
5487         int ret;
5488
5489         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5490         if (IS_ERR(mailbox))
5491                 return PTR_ERR(mailbox);
5492
5493         cq_context = mailbox->buf;
5494         cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
5495
5496         memset(cqc_mask, 0xff, sizeof(*cqc_mask));
5497
5498         hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count);
5499         hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT);
5500         hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);
5501         hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);
5502
5503         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
5504                                 HNS_ROCE_CMD_MODIFY_CQC,
5505                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
5506         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5507         if (ret)
5508                 ibdev_err(&hr_dev->ib_dev,
5509                           "failed to process cmd when modifying CQ, ret = %d.\n",
5510                           ret);
5511
5512         return ret;
5513 }
5514
5515 static void hns_roce_irq_work_handle(struct work_struct *work)
5516 {
5517         struct hns_roce_work *irq_work =
5518                                 container_of(work, struct hns_roce_work, work);
5519         struct ib_device *ibdev = &irq_work->hr_dev->ib_dev;
5520
5521         switch (irq_work->event_type) {
5522         case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5523                 ibdev_info(ibdev, "Path migrated succeeded.\n");
5524                 break;
5525         case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5526                 ibdev_warn(ibdev, "Path migration failed.\n");
5527                 break;
5528         case HNS_ROCE_EVENT_TYPE_COMM_EST:
5529                 break;
5530         case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5531                 ibdev_warn(ibdev, "Send queue drained.\n");
5532                 break;
5533         case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
5534                 ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n",
5535                           irq_work->queue_num, irq_work->sub_type);
5536                 break;
5537         case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5538                 ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n",
5539                           irq_work->queue_num);
5540                 break;
5541         case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5542                 ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n",
5543                           irq_work->queue_num, irq_work->sub_type);
5544                 break;
5545         case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5546                 ibdev_warn(ibdev, "SRQ limit reach.\n");
5547                 break;
5548         case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5549                 ibdev_warn(ibdev, "SRQ last wqe reach.\n");
5550                 break;
5551         case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5552                 ibdev_err(ibdev, "SRQ catas error.\n");
5553                 break;
5554         case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5555                 ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num);
5556                 break;
5557         case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5558                 ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num);
5559                 break;
5560         case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5561                 ibdev_warn(ibdev, "DB overflow.\n");
5562                 break;
5563         case HNS_ROCE_EVENT_TYPE_FLR:
5564                 ibdev_warn(ibdev, "Function level reset.\n");
5565                 break;
5566         case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
5567                 ibdev_err(ibdev, "xrc domain violation error.\n");
5568                 break;
5569         case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
5570                 ibdev_err(ibdev, "invalid xrceth error.\n");
5571                 break;
5572         default:
5573                 break;
5574         }
5575
5576         kfree(irq_work);
5577 }
5578
5579 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
5580                                       struct hns_roce_eq *eq, u32 queue_num)
5581 {
5582         struct hns_roce_work *irq_work;
5583
5584         irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
5585         if (!irq_work)
5586                 return;
5587
5588         INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
5589         irq_work->hr_dev = hr_dev;
5590         irq_work->event_type = eq->event_type;
5591         irq_work->sub_type = eq->sub_type;
5592         irq_work->queue_num = queue_num;
5593         queue_work(hr_dev->irq_workq, &(irq_work->work));
5594 }
5595
5596 static void update_eq_db(struct hns_roce_eq *eq)
5597 {
5598         struct hns_roce_dev *hr_dev = eq->hr_dev;
5599         struct hns_roce_v2_db eq_db = {};
5600
5601         if (eq->type_flag == HNS_ROCE_AEQ) {
5602                 hr_reg_write(&eq_db, EQ_DB_CMD,
5603                              eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
5604                              HNS_ROCE_EQ_DB_CMD_AEQ :
5605                              HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
5606         } else {
5607                 hr_reg_write(&eq_db, EQ_DB_TAG, eq->eqn);
5608
5609                 hr_reg_write(&eq_db, EQ_DB_CMD,
5610                              eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
5611                              HNS_ROCE_EQ_DB_CMD_CEQ :
5612                              HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
5613         }
5614
5615         hr_reg_write(&eq_db, EQ_DB_CI, eq->cons_index);
5616
5617         hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg);
5618 }
5619
5620 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
5621 {
5622         struct hns_roce_aeqe *aeqe;
5623
5624         aeqe = hns_roce_buf_offset(eq->mtr.kmem,
5625                                    (eq->cons_index & (eq->entries - 1)) *
5626                                    eq->eqe_size);
5627
5628         return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
5629                 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
5630 }
5631
5632 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
5633                                struct hns_roce_eq *eq)
5634 {
5635         struct device *dev = hr_dev->dev;
5636         struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
5637         int aeqe_found = 0;
5638         int event_type;
5639         u32 queue_num;
5640         int sub_type;
5641
5642         while (aeqe) {
5643                 /* Make sure we read AEQ entry after we have checked the
5644                  * ownership bit
5645                  */
5646                 dma_rmb();
5647
5648                 event_type = roce_get_field(aeqe->asyn,
5649                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
5650                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
5651                 sub_type = roce_get_field(aeqe->asyn,
5652                                           HNS_ROCE_V2_AEQE_SUB_TYPE_M,
5653                                           HNS_ROCE_V2_AEQE_SUB_TYPE_S);
5654                 queue_num = roce_get_field(aeqe->event.queue_event.num,
5655                                            HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5656                                            HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5657
5658                 switch (event_type) {
5659                 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5660                 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5661                 case HNS_ROCE_EVENT_TYPE_COMM_EST:
5662                 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5663                 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
5664                 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5665                 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5666                 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5667                 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
5668                 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
5669                         hns_roce_qp_event(hr_dev, queue_num, event_type);
5670                         break;
5671                 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5672                 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5673                         hns_roce_srq_event(hr_dev, queue_num, event_type);
5674                         break;
5675                 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5676                 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5677                         hns_roce_cq_event(hr_dev, queue_num, event_type);
5678                         break;
5679                 case HNS_ROCE_EVENT_TYPE_MB:
5680                         hns_roce_cmd_event(hr_dev,
5681                                         le16_to_cpu(aeqe->event.cmd.token),
5682                                         aeqe->event.cmd.status,
5683                                         le64_to_cpu(aeqe->event.cmd.out_param));
5684                         break;
5685                 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5686                 case HNS_ROCE_EVENT_TYPE_FLR:
5687                         break;
5688                 default:
5689                         dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
5690                                 event_type, eq->eqn, eq->cons_index);
5691                         break;
5692                 }
5693
5694                 eq->event_type = event_type;
5695                 eq->sub_type = sub_type;
5696                 ++eq->cons_index;
5697                 aeqe_found = 1;
5698
5699                 hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);
5700
5701                 aeqe = next_aeqe_sw_v2(eq);
5702         }
5703
5704         update_eq_db(eq);
5705         return aeqe_found;
5706 }
5707
5708 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5709 {
5710         struct hns_roce_ceqe *ceqe;
5711
5712         ceqe = hns_roce_buf_offset(eq->mtr.kmem,
5713                                    (eq->cons_index & (eq->entries - 1)) *
5714                                    eq->eqe_size);
5715
5716         return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5717                 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5718 }
5719
5720 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5721                                struct hns_roce_eq *eq)
5722 {
5723         struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
5724         int ceqe_found = 0;
5725         u32 cqn;
5726
5727         while (ceqe) {
5728                 /* Make sure we read CEQ entry after we have checked the
5729                  * ownership bit
5730                  */
5731                 dma_rmb();
5732
5733                 cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M,
5734                                      HNS_ROCE_V2_CEQE_COMP_CQN_S);
5735
5736                 hns_roce_cq_completion(hr_dev, cqn);
5737
5738                 ++eq->cons_index;
5739                 ceqe_found = 1;
5740
5741                 ceqe = next_ceqe_sw_v2(eq);
5742         }
5743
5744         update_eq_db(eq);
5745
5746         return ceqe_found;
5747 }
5748
5749 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5750 {
5751         struct hns_roce_eq *eq = eq_ptr;
5752         struct hns_roce_dev *hr_dev = eq->hr_dev;
5753         int int_work;
5754
5755         if (eq->type_flag == HNS_ROCE_CEQ)
5756                 /* Completion event interrupt */
5757                 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5758         else
5759                 /* Asychronous event interrupt */
5760                 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5761
5762         return IRQ_RETVAL(int_work);
5763 }
5764
5765 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5766 {
5767         struct hns_roce_dev *hr_dev = dev_id;
5768         struct device *dev = hr_dev->dev;
5769         int int_work = 0;
5770         u32 int_st;
5771         u32 int_en;
5772
5773         /* Abnormal interrupt */
5774         int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5775         int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5776
5777         if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
5778                 struct pci_dev *pdev = hr_dev->pci_dev;
5779                 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5780                 const struct hnae3_ae_ops *ops = ae_dev->ops;
5781
5782                 dev_err(dev, "AEQ overflow!\n");
5783
5784                 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S;
5785                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5786
5787                 /* Set reset level for reset_event() */
5788                 if (ops->set_default_reset_request)
5789                         ops->set_default_reset_request(ae_dev,
5790                                                        HNAE3_FUNC_RESET);
5791                 if (ops->reset_event)
5792                         ops->reset_event(pdev, NULL);
5793
5794                 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5795                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5796
5797                 int_work = 1;
5798         } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_RAS_INT_S)) {
5799                 dev_err(dev, "RAS interrupt!\n");
5800
5801                 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_RAS_INT_S;
5802                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5803
5804                 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5805                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5806
5807                 int_work = 1;
5808         } else {
5809                 dev_err(dev, "There is no abnormal irq found!\n");
5810         }
5811
5812         return IRQ_RETVAL(int_work);
5813 }
5814
5815 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5816                                         int eq_num, u32 enable_flag)
5817 {
5818         int i;
5819
5820         for (i = 0; i < eq_num; i++)
5821                 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5822                            i * EQ_REG_OFFSET, enable_flag);
5823
5824         roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag);
5825         roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);
5826 }
5827
5828 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5829 {
5830         struct device *dev = hr_dev->dev;
5831         int ret;
5832
5833         if (eqn < hr_dev->caps.num_comp_vectors)
5834                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5835                                         0, HNS_ROCE_CMD_DESTROY_CEQC,
5836                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5837         else
5838                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5839                                         0, HNS_ROCE_CMD_DESTROY_AEQC,
5840                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5841         if (ret)
5842                 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5843 }
5844
5845 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
5846 {
5847         hns_roce_mtr_destroy(hr_dev, &eq->mtr);
5848 }
5849
5850 static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
5851 {
5852         eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5853         eq->cons_index = 0;
5854         eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5855         eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5856         eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5857         eq->shift = ilog2((unsigned int)eq->entries);
5858 }
5859
5860 static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
5861                       void *mb_buf)
5862 {
5863         u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
5864         struct hns_roce_eq_context *eqc;
5865         u64 bt_ba = 0;
5866         int count;
5867
5868         eqc = mb_buf;
5869         memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5870
5871         init_eq_config(hr_dev, eq);
5872
5873         /* if not multi-hop, eqe buffer only use one trunk */
5874         count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT,
5875                                   &bt_ba);
5876         if (count < 1) {
5877                 dev_err(hr_dev->dev, "failed to find EQE mtr\n");
5878                 return -ENOBUFS;
5879         }
5880
5881         hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID);
5882         hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num);
5883         hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore);
5884         hr_reg_write(eqc, EQC_COALESCE, eq->coalesce);
5885         hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st);
5886         hr_reg_write(eqc, EQC_EQN, eq->eqn);
5887         hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT);
5888         hr_reg_write(eqc, EQC_EQE_BA_PG_SZ,
5889                      to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
5890         hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ,
5891                      to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
5892         hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX);
5893         hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt);
5894
5895         hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period);
5896         hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER);
5897         hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3);
5898         hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35);
5899         hr_reg_write(eqc, EQC_SHIFT, eq->shift);
5900         hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX);
5901         hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12);
5902         hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28);
5903         hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60);
5904         hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX);
5905         hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12);
5906         hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44);
5907         hr_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == HNS_ROCE_V3_EQE_SIZE);
5908
5909         return 0;
5910 }
5911
5912 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
5913 {
5914         struct hns_roce_buf_attr buf_attr = {};
5915         int err;
5916
5917         if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
5918                 eq->hop_num = 0;
5919         else
5920                 eq->hop_num = hr_dev->caps.eqe_hop_num;
5921
5922         buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT;
5923         buf_attr.region[0].size = eq->entries * eq->eqe_size;
5924         buf_attr.region[0].hopnum = eq->hop_num;
5925         buf_attr.region_count = 1;
5926
5927         err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
5928                                   hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
5929                                   0);
5930         if (err)
5931                 dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err);
5932
5933         return err;
5934 }
5935
5936 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5937                                  struct hns_roce_eq *eq,
5938                                  unsigned int eq_cmd)
5939 {
5940         struct hns_roce_cmd_mailbox *mailbox;
5941         int ret;
5942
5943         /* Allocate mailbox memory */
5944         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5945         if (IS_ERR_OR_NULL(mailbox))
5946                 return -ENOMEM;
5947
5948         ret = alloc_eq_buf(hr_dev, eq);
5949         if (ret)
5950                 goto free_cmd_mbox;
5951
5952         ret = config_eqc(hr_dev, eq, mailbox->buf);
5953         if (ret)
5954                 goto err_cmd_mbox;
5955
5956         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5957                                 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5958         if (ret) {
5959                 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
5960                 goto err_cmd_mbox;
5961         }
5962
5963         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5964
5965         return 0;
5966
5967 err_cmd_mbox:
5968         free_eq_buf(hr_dev, eq);
5969
5970 free_cmd_mbox:
5971         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5972
5973         return ret;
5974 }
5975
5976 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
5977                                   int comp_num, int aeq_num, int other_num)
5978 {
5979         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5980         int i, j;
5981         int ret;
5982
5983         for (i = 0; i < irq_num; i++) {
5984                 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5985                                                GFP_KERNEL);
5986                 if (!hr_dev->irq_names[i]) {
5987                         ret = -ENOMEM;
5988                         goto err_kzalloc_failed;
5989                 }
5990         }
5991
5992         /* irq contains: abnormal + AEQ + CEQ */
5993         for (j = 0; j < other_num; j++)
5994                 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
5995                          "hns-abn-%d", j);
5996
5997         for (j = other_num; j < (other_num + aeq_num); j++)
5998                 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
5999                          "hns-aeq-%d", j - other_num);
6000
6001         for (j = (other_num + aeq_num); j < irq_num; j++)
6002                 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6003                          "hns-ceq-%d", j - other_num - aeq_num);
6004
6005         for (j = 0; j < irq_num; j++) {
6006                 if (j < other_num)
6007                         ret = request_irq(hr_dev->irq[j],
6008                                           hns_roce_v2_msix_interrupt_abn,
6009                                           0, hr_dev->irq_names[j], hr_dev);
6010
6011                 else if (j < (other_num + comp_num))
6012                         ret = request_irq(eq_table->eq[j - other_num].irq,
6013                                           hns_roce_v2_msix_interrupt_eq,
6014                                           0, hr_dev->irq_names[j + aeq_num],
6015                                           &eq_table->eq[j - other_num]);
6016                 else
6017                         ret = request_irq(eq_table->eq[j - other_num].irq,
6018                                           hns_roce_v2_msix_interrupt_eq,
6019                                           0, hr_dev->irq_names[j - comp_num],
6020                                           &eq_table->eq[j - other_num]);
6021                 if (ret) {
6022                         dev_err(hr_dev->dev, "Request irq error!\n");
6023                         goto err_request_failed;
6024                 }
6025         }
6026
6027         return 0;
6028
6029 err_request_failed:
6030         for (j -= 1; j >= 0; j--)
6031                 if (j < other_num)
6032                         free_irq(hr_dev->irq[j], hr_dev);
6033                 else
6034                         free_irq(eq_table->eq[j - other_num].irq,
6035                                  &eq_table->eq[j - other_num]);
6036
6037 err_kzalloc_failed:
6038         for (i -= 1; i >= 0; i--)
6039                 kfree(hr_dev->irq_names[i]);
6040
6041         return ret;
6042 }
6043
6044 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
6045 {
6046         int irq_num;
6047         int eq_num;
6048         int i;
6049
6050         eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6051         irq_num = eq_num + hr_dev->caps.num_other_vectors;
6052
6053         for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
6054                 free_irq(hr_dev->irq[i], hr_dev);
6055
6056         for (i = 0; i < eq_num; i++)
6057                 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
6058
6059         for (i = 0; i < irq_num; i++)
6060                 kfree(hr_dev->irq_names[i]);
6061 }
6062
6063 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
6064 {
6065         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6066         struct device *dev = hr_dev->dev;
6067         struct hns_roce_eq *eq;
6068         unsigned int eq_cmd;
6069         int irq_num;
6070         int eq_num;
6071         int other_num;
6072         int comp_num;
6073         int aeq_num;
6074         int i;
6075         int ret;
6076
6077         other_num = hr_dev->caps.num_other_vectors;
6078         comp_num = hr_dev->caps.num_comp_vectors;
6079         aeq_num = hr_dev->caps.num_aeq_vectors;
6080
6081         eq_num = comp_num + aeq_num;
6082         irq_num = eq_num + other_num;
6083
6084         eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
6085         if (!eq_table->eq)
6086                 return -ENOMEM;
6087
6088         /* create eq */
6089         for (i = 0; i < eq_num; i++) {
6090                 eq = &eq_table->eq[i];
6091                 eq->hr_dev = hr_dev;
6092                 eq->eqn = i;
6093                 if (i < comp_num) {
6094                         /* CEQ */
6095                         eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
6096                         eq->type_flag = HNS_ROCE_CEQ;
6097                         eq->entries = hr_dev->caps.ceqe_depth;
6098                         eq->eqe_size = hr_dev->caps.ceqe_size;
6099                         eq->irq = hr_dev->irq[i + other_num + aeq_num];
6100                         eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
6101                         eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
6102                 } else {
6103                         /* AEQ */
6104                         eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
6105                         eq->type_flag = HNS_ROCE_AEQ;
6106                         eq->entries = hr_dev->caps.aeqe_depth;
6107                         eq->eqe_size = hr_dev->caps.aeqe_size;
6108                         eq->irq = hr_dev->irq[i - comp_num + other_num];
6109                         eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
6110                         eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
6111                 }
6112
6113                 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
6114                 if (ret) {
6115                         dev_err(dev, "eq create failed.\n");
6116                         goto err_create_eq_fail;
6117                 }
6118         }
6119
6120         /* enable irq */
6121         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
6122
6123         ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
6124                                      aeq_num, other_num);
6125         if (ret) {
6126                 dev_err(dev, "Request irq failed.\n");
6127                 goto err_request_irq_fail;
6128         }
6129
6130         hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);
6131         if (!hr_dev->irq_workq) {
6132                 dev_err(dev, "Create irq workqueue failed!\n");
6133                 ret = -ENOMEM;
6134                 goto err_create_wq_fail;
6135         }
6136
6137         return 0;
6138
6139 err_create_wq_fail:
6140         __hns_roce_free_irq(hr_dev);
6141
6142 err_request_irq_fail:
6143         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6144
6145 err_create_eq_fail:
6146         for (i -= 1; i >= 0; i--)
6147                 free_eq_buf(hr_dev, &eq_table->eq[i]);
6148         kfree(eq_table->eq);
6149
6150         return ret;
6151 }
6152
6153 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
6154 {
6155         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6156         int eq_num;
6157         int i;
6158
6159         eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6160
6161         /* Disable irq */
6162         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6163
6164         __hns_roce_free_irq(hr_dev);
6165         destroy_workqueue(hr_dev->irq_workq);
6166
6167         for (i = 0; i < eq_num; i++) {
6168                 hns_roce_v2_destroy_eqc(hr_dev, i);
6169
6170                 free_eq_buf(hr_dev, &eq_table->eq[i]);
6171         }
6172
6173         kfree(eq_table->eq);
6174 }
6175
6176 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6177         .query_cqc_info = hns_roce_v2_query_cqc_info,
6178 };
6179
6180 static const struct ib_device_ops hns_roce_v2_dev_ops = {
6181         .destroy_qp = hns_roce_v2_destroy_qp,
6182         .modify_cq = hns_roce_v2_modify_cq,
6183         .poll_cq = hns_roce_v2_poll_cq,
6184         .post_recv = hns_roce_v2_post_recv,
6185         .post_send = hns_roce_v2_post_send,
6186         .query_qp = hns_roce_v2_query_qp,
6187         .req_notify_cq = hns_roce_v2_req_notify_cq,
6188 };
6189
6190 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6191         .modify_srq = hns_roce_v2_modify_srq,
6192         .post_srq_recv = hns_roce_v2_post_srq_recv,
6193         .query_srq = hns_roce_v2_query_srq,
6194 };
6195
6196 static const struct hns_roce_hw hns_roce_hw_v2 = {
6197         .cmq_init = hns_roce_v2_cmq_init,
6198         .cmq_exit = hns_roce_v2_cmq_exit,
6199         .hw_profile = hns_roce_v2_profile,
6200         .hw_init = hns_roce_v2_init,
6201         .hw_exit = hns_roce_v2_exit,
6202         .post_mbox = v2_post_mbox,
6203         .poll_mbox_done = v2_poll_mbox_done,
6204         .chk_mbox_avail = v2_chk_mbox_is_avail,
6205         .set_gid = hns_roce_v2_set_gid,
6206         .set_mac = hns_roce_v2_set_mac,
6207         .write_mtpt = hns_roce_v2_write_mtpt,
6208         .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
6209         .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
6210         .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
6211         .write_cqc = hns_roce_v2_write_cqc,
6212         .set_hem = hns_roce_v2_set_hem,
6213         .clear_hem = hns_roce_v2_clear_hem,
6214         .modify_qp = hns_roce_v2_modify_qp,
6215         .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
6216         .init_eq = hns_roce_v2_init_eq_table,
6217         .cleanup_eq = hns_roce_v2_cleanup_eq_table,
6218         .write_srqc = hns_roce_v2_write_srqc,
6219         .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6220         .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
6221 };
6222
6223 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6224         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6225         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
6226         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6227         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
6228         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6229         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
6230         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
6231          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
6232         /* required last entry */
6233         {0, }
6234 };
6235
6236 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6237
6238 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6239                                   struct hnae3_handle *handle)
6240 {
6241         struct hns_roce_v2_priv *priv = hr_dev->priv;
6242         const struct pci_device_id *id;
6243         int i;
6244
6245         hr_dev->pci_dev = handle->pdev;
6246         id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
6247         hr_dev->is_vf = id->driver_data;
6248         hr_dev->dev = &handle->pdev->dev;
6249         hr_dev->hw = &hns_roce_hw_v2;
6250         hr_dev->dfx = &hns_roce_dfx_hw_v2;
6251         hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6252         hr_dev->odb_offset = hr_dev->sdb_offset;
6253
6254         /* Get info from NIC driver. */
6255         hr_dev->reg_base = handle->rinfo.roce_io_base;
6256         hr_dev->mem_base = handle->rinfo.roce_mem_base;
6257         hr_dev->caps.num_ports = 1;
6258         hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6259         hr_dev->iboe.phy_port[0] = 0;
6260
6261         addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6262                             hr_dev->iboe.netdevs[0]->dev_addr);
6263
6264         for (i = 0; i < handle->rinfo.num_vectors; i++)
6265                 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6266                                                 i + handle->rinfo.base_vector);
6267
6268         /* cmd issue mode: 0 is poll, 1 is event */
6269         hr_dev->cmd_mod = 1;
6270         hr_dev->loop_idc = 0;
6271
6272         hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6273         priv->handle = handle;
6274 }
6275
6276 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6277 {
6278         struct hns_roce_dev *hr_dev;
6279         int ret;
6280
6281         hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
6282         if (!hr_dev)
6283                 return -ENOMEM;
6284
6285         hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6286         if (!hr_dev->priv) {
6287                 ret = -ENOMEM;
6288                 goto error_failed_kzalloc;
6289         }
6290
6291         hns_roce_hw_v2_get_cfg(hr_dev, handle);
6292
6293         ret = hns_roce_init(hr_dev);
6294         if (ret) {
6295                 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6296                 goto error_failed_get_cfg;
6297         }
6298
6299         handle->priv = hr_dev;
6300
6301         return 0;
6302
6303 error_failed_get_cfg:
6304         kfree(hr_dev->priv);
6305
6306 error_failed_kzalloc:
6307         ib_dealloc_device(&hr_dev->ib_dev);
6308
6309         return ret;
6310 }
6311
6312 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6313                                            bool reset)
6314 {
6315         struct hns_roce_dev *hr_dev = handle->priv;
6316
6317         if (!hr_dev)
6318                 return;
6319
6320         handle->priv = NULL;
6321
6322         hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
6323         hns_roce_handle_device_err(hr_dev);
6324
6325         hns_roce_exit(hr_dev);
6326         kfree(hr_dev->priv);
6327         ib_dealloc_device(&hr_dev->ib_dev);
6328 }
6329
6330 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6331 {
6332         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
6333         const struct pci_device_id *id;
6334         struct device *dev = &handle->pdev->dev;
6335         int ret;
6336
6337         handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6338
6339         if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6340                 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6341                 goto reset_chk_err;
6342         }
6343
6344         id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6345         if (!id)
6346                 return 0;
6347
6348         if (id->driver_data && handle->pdev->revision < PCI_REVISION_ID_HIP09)
6349                 return 0;
6350
6351         ret = __hns_roce_hw_v2_init_instance(handle);
6352         if (ret) {
6353                 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6354                 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6355                 if (ops->ae_dev_resetting(handle) ||
6356                     ops->get_hw_reset_stat(handle))
6357                         goto reset_chk_err;
6358                 else
6359                         return ret;
6360         }
6361
6362         handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6363
6364
6365         return 0;
6366
6367 reset_chk_err:
6368         dev_err(dev, "Device is busy in resetting state.\n"
6369                      "please retry later.\n");
6370
6371         return -EBUSY;
6372 }
6373
6374 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6375                                            bool reset)
6376 {
6377         if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6378                 return;
6379
6380         handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6381
6382         __hns_roce_hw_v2_uninit_instance(handle, reset);
6383
6384         handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6385 }
6386 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6387 {
6388         struct hns_roce_dev *hr_dev;
6389
6390         if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6391                 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6392                 return 0;
6393         }
6394
6395         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6396         clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6397
6398         hr_dev = handle->priv;
6399         if (!hr_dev)
6400                 return 0;
6401
6402         hr_dev->is_reset = true;
6403         hr_dev->active = false;
6404         hr_dev->dis_db = true;
6405
6406         hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
6407
6408         return 0;
6409 }
6410
6411 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6412 {
6413         struct device *dev = &handle->pdev->dev;
6414         int ret;
6415
6416         if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6417                                &handle->rinfo.state)) {
6418                 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6419                 return 0;
6420         }
6421
6422         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6423
6424         dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6425         ret = __hns_roce_hw_v2_init_instance(handle);
6426         if (ret) {
6427                 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6428                  * callback function, RoCE Engine reinitialize. If RoCE reinit
6429                  * failed, we should inform NIC driver.
6430                  */
6431                 handle->priv = NULL;
6432                 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6433         } else {
6434                 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6435                 dev_info(dev, "Reset done, RoCE client reinit finished.\n");
6436         }
6437
6438         return ret;
6439 }
6440
6441 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6442 {
6443         if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6444                 return 0;
6445
6446         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6447         dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
6448         msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
6449         __hns_roce_hw_v2_uninit_instance(handle, false);
6450
6451         return 0;
6452 }
6453
6454 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6455                                        enum hnae3_reset_notify_type type)
6456 {
6457         int ret = 0;
6458
6459         switch (type) {
6460         case HNAE3_DOWN_CLIENT:
6461                 ret = hns_roce_hw_v2_reset_notify_down(handle);
6462                 break;
6463         case HNAE3_INIT_CLIENT:
6464                 ret = hns_roce_hw_v2_reset_notify_init(handle);
6465                 break;
6466         case HNAE3_UNINIT_CLIENT:
6467                 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6468                 break;
6469         default:
6470                 break;
6471         }
6472
6473         return ret;
6474 }
6475
6476 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6477         .init_instance = hns_roce_hw_v2_init_instance,
6478         .uninit_instance = hns_roce_hw_v2_uninit_instance,
6479         .reset_notify = hns_roce_hw_v2_reset_notify,
6480 };
6481
6482 static struct hnae3_client hns_roce_hw_v2_client = {
6483         .name = "hns_roce_hw_v2",
6484         .type = HNAE3_CLIENT_ROCE,
6485         .ops = &hns_roce_hw_v2_ops,
6486 };
6487
6488 static int __init hns_roce_hw_v2_init(void)
6489 {
6490         return hnae3_register_client(&hns_roce_hw_v2_client);
6491 }
6492
6493 static void __exit hns_roce_hw_v2_exit(void)
6494 {
6495         hnae3_unregister_client(&hns_roce_hw_v2_client);
6496 }
6497
6498 module_init(hns_roce_hw_v2_init);
6499 module_exit(hns_roce_hw_v2_exit);
6500
6501 MODULE_LICENSE("Dual BSD/GPL");
6502 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
6503 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
6504 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
6505 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");