drivers/infiniband/hw/hns/hns_roce_qp.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/pci.h>
  35 #include <linux/platform_device.h>
  36 #include <rdma/ib_addr.h>
  37 #include <rdma/ib_umem.h>
  38 #include <rdma/uverbs_ioctl.h>
  39 #include "hns_roce_common.h"
  40 #include "hns_roce_device.h"
  41 #include "hns_roce_hem.h"
  42 #include <rdma/hns-abi.h>
  43
  44 #define SQP_NUM                         (2 * HNS_ROCE_MAX_PORTS)
  45
  46 static void flush_work_handle(struct work_struct *work)
  47 {
  48         struct hns_roce_work *flush_work = container_of(work,
  49                                         struct hns_roce_work, work);
  50         struct hns_roce_qp *hr_qp = container_of(flush_work,
  51                                         struct hns_roce_qp, flush_work);
  52         struct device *dev = flush_work->hr_dev->dev;
  53         struct ib_qp_attr attr;
  54         int attr_mask;
  55         int ret;
  56
  57         attr_mask = IB_QP_STATE;
  58         attr.qp_state = IB_QPS_ERR;
  59
  60         if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) {
  61                 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL);
  62                 if (ret)
  63                         dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n",
  64                                 ret);
  65         }
  66
  67         /*
  68          * make sure we signal QP destroy leg that flush QP was completed
  69          * so that it can safely proceed ahead now and destroy QP
  70          */
  71         if (atomic_dec_and_test(&hr_qp->refcount))
  72                 complete(&hr_qp->free);
  73 }
  74
  75 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
  76 {
  77         struct hns_roce_work *flush_work = &hr_qp->flush_work;
  78
  79         flush_work->hr_dev = hr_dev;
  80         INIT_WORK(&flush_work->work, flush_work_handle);
  81         atomic_inc(&hr_qp->refcount);
  82         queue_work(hr_dev->irq_workq, &flush_work->work);
  83 }
  84
  85 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
  86 {
  87         struct device *dev = hr_dev->dev;
  88         struct hns_roce_qp *qp;
  89
  90         xa_lock(&hr_dev->qp_table_xa);
  91         qp = __hns_roce_qp_lookup(hr_dev, qpn);
  92         if (qp)
  93                 atomic_inc(&qp->refcount);
  94         xa_unlock(&hr_dev->qp_table_xa);
  95
  96         if (!qp) {
  97                 dev_warn(dev, "Async event for bogus QP %08x\n", qpn);
  98                 return;
  99         }
 100
 101         if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 &&
 102             (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR ||
 103              event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR ||
 104              event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR)) {
 105                 qp->state = IB_QPS_ERR;
 106                 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
 107                         init_flush_work(hr_dev, qp);
 108         }
 109
 110         qp->event(qp, (enum hns_roce_event)event_type);
 111
 112         if (atomic_dec_and_test(&qp->refcount))
 113                 complete(&qp->free);
 114 }
 115
 116 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
 117                                  enum hns_roce_event type)
 118 {
 119         struct ib_event event;
 120         struct ib_qp *ibqp = &hr_qp->ibqp;
 121
 122         if (ibqp->event_handler) {
 123                 event.device = ibqp->device;
 124                 event.element.qp = ibqp;
 125                 switch (type) {
 126                 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
 127                         event.event = IB_EVENT_PATH_MIG;
 128                         break;
 129                 case HNS_ROCE_EVENT_TYPE_COMM_EST:
 130                         event.event = IB_EVENT_COMM_EST;
 131                         break;
 132                 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
 133                         event.event = IB_EVENT_SQ_DRAINED;
 134                         break;
 135                 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
 136                         event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 137                         break;
 138                 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
 139                         event.event = IB_EVENT_QP_FATAL;
 140                         break;
 141                 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
 142                         event.event = IB_EVENT_PATH_MIG_ERR;
 143                         break;
 144                 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
 145                         event.event = IB_EVENT_QP_REQ_ERR;
 146                         break;
 147                 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
 148                         event.event = IB_EVENT_QP_ACCESS_ERR;
 149                         break;
 150                 default:
 151                         dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
 152                                 type, hr_qp->qpn);
 153                         return;
 154                 }
 155                 ibqp->event_handler(&event, ibqp->qp_context);
 156         }
 157 }
 158
 159 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 160 {
 161         unsigned long num = 0;
 162         int ret;
 163
 164         if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
 165                 /* when hw version is v1, the sqpn is allocated */
 166                 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
 167                         num = HNS_ROCE_MAX_PORTS +
 168                               hr_dev->iboe.phy_port[hr_qp->port];
 169                 else
 170                         num = 1;
 171
 172                 hr_qp->doorbell_qpn = 1;
 173         } else {
 174                 ret = hns_roce_bitmap_alloc_range(&hr_dev->qp_table.bitmap,
 175                                                   1, 1, &num);
 176                 if (ret) {
 177                         ibdev_err(&hr_dev->ib_dev, "Failed to alloc bitmap\n");
 178                         return -ENOMEM;
 179                 }
 180
 181                 hr_qp->doorbell_qpn = (u32)num;
 182         }
 183
 184         hr_qp->qpn = num;
 185
 186         return 0;
 187 }
 188
 189 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state)
 190 {
 191         switch (state) {
 192         case IB_QPS_RESET:
 193                 return HNS_ROCE_QP_STATE_RST;
 194         case IB_QPS_INIT:
 195                 return HNS_ROCE_QP_STATE_INIT;
 196         case IB_QPS_RTR:
 197                 return HNS_ROCE_QP_STATE_RTR;
 198         case IB_QPS_RTS:
 199                 return HNS_ROCE_QP_STATE_RTS;
 200         case IB_QPS_SQD:
 201                 return HNS_ROCE_QP_STATE_SQD;
 202         case IB_QPS_ERR:
 203                 return HNS_ROCE_QP_STATE_ERR;
 204         default:
 205                 return HNS_ROCE_QP_NUM_STATE;
 206         }
 207 }
 208
 209 static void add_qp_to_list(struct hns_roce_dev *hr_dev,
 210                            struct hns_roce_qp *hr_qp,
 211                            struct ib_cq *send_cq, struct ib_cq *recv_cq)
 212 {
 213         struct hns_roce_cq *hr_send_cq, *hr_recv_cq;
 214         unsigned long flags;
 215
 216         hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL;
 217         hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL;
 218
 219         spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
 220         hns_roce_lock_cqs(hr_send_cq, hr_recv_cq);
 221
 222         list_add_tail(&hr_qp->node, &hr_dev->qp_list);
 223         if (hr_send_cq)
 224                 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list);
 225         if (hr_recv_cq)
 226                 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list);
 227
 228         hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq);
 229         spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
 230 }
 231
 232 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev,
 233                              struct hns_roce_qp *hr_qp,
 234                              struct ib_qp_init_attr *init_attr)
 235 {
 236         struct xarray *xa = &hr_dev->qp_table_xa;
 237         int ret;
 238
 239         if (!hr_qp->qpn)
 240                 return -EINVAL;
 241
 242         ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL));
 243         if (ret)
 244                 dev_err(hr_dev->dev, "Failed to xa store for QPC\n");
 245         else
 246                 /* add QP to device's QP list for softwc */
 247                 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq,
 248                                init_attr->recv_cq);
 249
 250         return ret;
 251 }
 252
 253 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 254 {
 255         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 256         struct device *dev = hr_dev->dev;
 257         int ret;
 258
 259         if (!hr_qp->qpn)
 260                 return -EINVAL;
 261
 262         /* In v1 engine, GSI QP context is saved in the RoCE hw's register */
 263         if (hr_qp->ibqp.qp_type == IB_QPT_GSI &&
 264             hr_dev->hw_rev == HNS_ROCE_HW_VER1)
 265                 return 0;
 266
 267         /* Alloc memory for QPC */
 268         ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
 269         if (ret) {
 270                 dev_err(dev, "Failed to get QPC table\n");
 271                 goto err_out;
 272         }
 273
 274         /* Alloc memory for IRRL */
 275         ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
 276         if (ret) {
 277                 dev_err(dev, "Failed to get IRRL table\n");
 278                 goto err_put_qp;
 279         }
 280
 281         if (hr_dev->caps.trrl_entry_sz) {
 282                 /* Alloc memory for TRRL */
 283                 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
 284                                          hr_qp->qpn);
 285                 if (ret) {
 286                         dev_err(dev, "Failed to get TRRL table\n");
 287                         goto err_put_irrl;
 288                 }
 289         }
 290
 291         if (hr_dev->caps.sccc_entry_sz) {
 292                 /* Alloc memory for SCC CTX */
 293                 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
 294                                          hr_qp->qpn);
 295                 if (ret) {
 296                         dev_err(dev, "Failed to get SCC CTX table\n");
 297                         goto err_put_trrl;
 298                 }
 299         }
 300
 301         return 0;
 302
 303 err_put_trrl:
 304         if (hr_dev->caps.trrl_entry_sz)
 305                 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
 306
 307 err_put_irrl:
 308         hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
 309
 310 err_put_qp:
 311         hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
 312
 313 err_out:
 314         return ret;
 315 }
 316
 317 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 318 {
 319         struct xarray *xa = &hr_dev->qp_table_xa;
 320         unsigned long flags;
 321
 322         list_del(&hr_qp->node);
 323         list_del(&hr_qp->sq_node);
 324         list_del(&hr_qp->rq_node);
 325
 326         xa_lock_irqsave(xa, flags);
 327         __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1));
 328         xa_unlock_irqrestore(xa, flags);
 329 }
 330
 331 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 332 {
 333         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 334
 335         /* In v1 engine, GSI QP context is saved in the RoCE hw's register */
 336         if (hr_qp->ibqp.qp_type == IB_QPT_GSI &&
 337             hr_dev->hw_rev == HNS_ROCE_HW_VER1)
 338                 return;
 339
 340         if (hr_dev->caps.trrl_entry_sz)
 341                 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
 342         hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
 343 }
 344
 345 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 346 {
 347         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 348
 349         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
 350                 return;
 351
 352         if (hr_qp->qpn < hr_dev->caps.reserved_qps)
 353                 return;
 354
 355         hns_roce_bitmap_free_range(&qp_table->bitmap, hr_qp->qpn, 1, BITMAP_RR);
 356 }
 357
 358 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
 359                        struct hns_roce_qp *hr_qp, int has_rq)
 360 {
 361         u32 cnt;
 362
 363         /* If srq exist, set zero for relative number of rq */
 364         if (!has_rq) {
 365                 hr_qp->rq.wqe_cnt = 0;
 366                 hr_qp->rq.max_gs = 0;
 367                 hr_qp->rq_inl_buf.wqe_cnt = 0;
 368                 cap->max_recv_wr = 0;
 369                 cap->max_recv_sge = 0;
 370
 371                 return 0;
 372         }
 373
 374         /* Check the validity of QP support capacity */
 375         if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes ||
 376             cap->max_recv_sge > hr_dev->caps.max_rq_sg) {
 377                 ibdev_err(&hr_dev->ib_dev, "RQ config error, depth=%u, sge=%d\n",
 378                           cap->max_recv_wr, cap->max_recv_sge);
 379                 return -EINVAL;
 380         }
 381
 382         cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes));
 383         if (cnt > hr_dev->caps.max_wqes) {
 384                 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n",
 385                           cap->max_recv_wr);
 386                 return -EINVAL;
 387         }
 388
 389         hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) +
 390                                               HNS_ROCE_RESERVED_SGE);
 391
 392         if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE)
 393                 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz);
 394         else
 395                 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz *
 396                                             hr_qp->rq.max_gs);
 397
 398         hr_qp->rq.wqe_cnt = cnt;
 399         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE)
 400                 hr_qp->rq_inl_buf.wqe_cnt = cnt;
 401         else
 402                 hr_qp->rq_inl_buf.wqe_cnt = 0;
 403
 404         cap->max_recv_wr = cnt;
 405         cap->max_recv_sge = hr_qp->rq.max_gs - HNS_ROCE_RESERVED_SGE;
 406
 407         return 0;
 408 }
 409
 410 static int set_extend_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
 411                                 struct hns_roce_qp *hr_qp,
 412                                 struct ib_qp_cap *cap)
 413 {
 414         u32 cnt;
 415
 416         cnt = max(1U, cap->max_send_sge);
 417         if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
 418                 hr_qp->sq.max_gs = roundup_pow_of_two(cnt);
 419                 hr_qp->sge.sge_cnt = 0;
 420
 421                 return 0;
 422         }
 423
 424         hr_qp->sq.max_gs = cnt;
 425
 426         /* UD sqwqe's sge use extend sge */
 427         if (hr_qp->ibqp.qp_type == IB_QPT_GSI ||
 428             hr_qp->ibqp.qp_type == IB_QPT_UD) {
 429                 cnt = roundup_pow_of_two(sq_wqe_cnt * hr_qp->sq.max_gs);
 430         } else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) {
 431                 cnt = roundup_pow_of_two(sq_wqe_cnt *
 432                                      (hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE));
 433         } else {
 434                 cnt = 0;
 435         }
 436
 437         hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
 438         hr_qp->sge.sge_cnt = cnt;
 439
 440         return 0;
 441 }
 442
 443 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
 444                                         struct ib_qp_cap *cap,
 445                                         struct hns_roce_ib_create_qp *ucmd)
 446 {
 447         u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
 448         u8 max_sq_stride = ilog2(roundup_sq_stride);
 449
 450         /* Sanity check SQ size before proceeding */
 451         if (ucmd->log_sq_stride > max_sq_stride ||
 452             ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
 453                 ibdev_err(&hr_dev->ib_dev, "Failed to check SQ stride size\n");
 454                 return -EINVAL;
 455         }
 456
 457         if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
 458                 ibdev_err(&hr_dev->ib_dev, "Failed to check SQ SGE size %d\n",
 459                           cap->max_send_sge);
 460                 return -EINVAL;
 461         }
 462
 463         return 0;
 464 }
 465
 466 static int set_user_sq_size(struct hns_roce_dev *hr_dev,
 467                             struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,
 468                             struct hns_roce_ib_create_qp *ucmd)
 469 {
 470         struct ib_device *ibdev = &hr_dev->ib_dev;
 471         u32 cnt = 0;
 472         int ret;
 473
 474         if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
 475             cnt > hr_dev->caps.max_wqes)
 476                 return -EINVAL;
 477
 478         ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
 479         if (ret) {
 480                 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",
 481                           ret);
 482                 return ret;
 483         }
 484
 485         ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap);
 486         if (ret)
 487                 return ret;
 488
 489         hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
 490         hr_qp->sq.wqe_cnt = cnt;
 491
 492         return 0;
 493 }
 494
 495 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
 496                             struct hns_roce_qp *hr_qp,
 497                             struct hns_roce_buf_attr *buf_attr)
 498 {
 499         int buf_size;
 500         int idx = 0;
 501
 502         hr_qp->buff_size = 0;
 503
 504         /* SQ WQE */
 505         hr_qp->sq.offset = 0;
 506         buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt,
 507                                           hr_qp->sq.wqe_shift);
 508         if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
 509                 buf_attr->region[idx].size = buf_size;
 510                 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num;
 511                 idx++;
 512                 hr_qp->buff_size += buf_size;
 513         }
 514
 515         /* extend SGE WQE in SQ */
 516         hr_qp->sge.offset = hr_qp->buff_size;
 517         buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt,
 518                                           hr_qp->sge.sge_shift);
 519         if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
 520                 buf_attr->region[idx].size = buf_size;
 521                 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num;
 522                 idx++;
 523                 hr_qp->buff_size += buf_size;
 524         }
 525
 526         /* RQ WQE */
 527         hr_qp->rq.offset = hr_qp->buff_size;
 528         buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt,
 529                                           hr_qp->rq.wqe_shift);
 530         if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
 531                 buf_attr->region[idx].size = buf_size;
 532                 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num;
 533                 idx++;
 534                 hr_qp->buff_size += buf_size;
 535         }
 536
 537         if (hr_qp->buff_size < 1)
 538                 return -EINVAL;
 539
 540         buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
 541         buf_attr->fixed_page = true;
 542         buf_attr->region_count = idx;
 543
 544         return 0;
 545 }
 546
 547 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,
 548                               struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)
 549 {
 550         struct ib_device *ibdev = &hr_dev->ib_dev;
 551         u32 cnt;
 552         int ret;
 553
 554         if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
 555             cap->max_send_sge > hr_dev->caps.max_sq_sg ||
 556             cap->max_inline_data > hr_dev->caps.max_sq_inline) {
 557                 ibdev_err(ibdev,
 558                           "failed to check SQ WR, SGE or inline num, ret = %d.\n",
 559                           -EINVAL);
 560                 return -EINVAL;
 561         }
 562
 563         cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
 564         if (cnt > hr_dev->caps.max_wqes) {
 565                 ibdev_err(ibdev, "failed to check WQE num, WQE num = %d.\n",
 566                           cnt);
 567                 return -EINVAL;
 568         }
 569
 570         hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
 571         hr_qp->sq.wqe_cnt = cnt;
 572
 573         ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap);
 574         if (ret)
 575                 return ret;
 576
 577         /* sync the parameters of kernel QP to user's configuration */
 578         cap->max_send_wr = cnt;
 579         cap->max_send_sge = hr_qp->sq.max_gs;
 580
 581         /* We don't support inline sends for kernel QPs (yet) */
 582         cap->max_inline_data = 0;
 583
 584         return 0;
 585 }
 586
 587 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
 588 {
 589         if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
 590                 return 0;
 591
 592         return 1;
 593 }
 594
 595 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
 596 {
 597         if (attr->qp_type == IB_QPT_XRC_INI ||
 598             attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
 599             !attr->cap.max_recv_wr)
 600                 return 0;
 601
 602         return 1;
 603 }
 604
 605 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
 606                                struct ib_qp_init_attr *init_attr)
 607 {
 608         u32 max_recv_sge = init_attr->cap.max_recv_sge;
 609         u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt;
 610         struct hns_roce_rinl_wqe *wqe_list;
 611         int i;
 612
 613         /* allocate recv inline buf */
 614         wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
 615                            GFP_KERNEL);
 616
 617         if (!wqe_list)
 618                 goto err;
 619
 620         /* Allocate a continuous buffer for all inline sge we need */
 621         wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
 622                                       sizeof(struct hns_roce_rinl_sge)),
 623                                       GFP_KERNEL);
 624         if (!wqe_list[0].sg_list)
 625                 goto err_wqe_list;
 626
 627         /* Assign buffers of sg_list to each inline wqe */
 628         for (i = 1; i < wqe_cnt; i++)
 629                 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
 630
 631         hr_qp->rq_inl_buf.wqe_list = wqe_list;
 632
 633         return 0;
 634
 635 err_wqe_list:
 636         kfree(wqe_list);
 637
 638 err:
 639         return -ENOMEM;
 640 }
 641
 642 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
 643 {
 644         if (hr_qp->rq_inl_buf.wqe_list)
 645                 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
 646         kfree(hr_qp->rq_inl_buf.wqe_list);
 647 }
 648
 649 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 650                         struct ib_qp_init_attr *init_attr,
 651                         struct ib_udata *udata, unsigned long addr)
 652 {
 653         struct ib_device *ibdev = &hr_dev->ib_dev;
 654         struct hns_roce_buf_attr buf_attr = {};
 655         int ret;
 656
 657         if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
 658                 ret = alloc_rq_inline_buf(hr_qp, init_attr);
 659                 if (ret) {
 660                         ibdev_err(ibdev,
 661                                   "failed to alloc inline buf, ret = %d.\n",
 662                                   ret);
 663                         return ret;
 664                 }
 665         } else {
 666                 hr_qp->rq_inl_buf.wqe_list = NULL;
 667         }
 668
 669         ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
 670         if (ret) {
 671                 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
 672                 goto err_inline;
 673         }
 674         ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
 675                                   HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
 676                                   udata, addr);
 677         if (ret) {
 678                 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
 679                 goto err_inline;
 680         }
 681
 682         return 0;
 683 err_inline:
 684         free_rq_inline_buf(hr_qp);
 685
 686         return ret;
 687 }
 688
 689 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 690 {
 691         hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
 692         free_rq_inline_buf(hr_qp);
 693 }
 694
 695 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
 696                                    struct ib_qp_init_attr *init_attr,
 697                                    struct ib_udata *udata,
 698                                    struct hns_roce_ib_create_qp_resp *resp,
 699                                    struct hns_roce_ib_create_qp *ucmd)
 700 {
 701         return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &&
 702                 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
 703                 hns_roce_qp_has_sq(init_attr) &&
 704                 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr));
 705 }
 706
 707 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev,
 708                                    struct ib_qp_init_attr *init_attr,
 709                                    struct ib_udata *udata,
 710                                    struct hns_roce_ib_create_qp_resp *resp)
 711 {
 712         return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
 713                 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
 714                 hns_roce_qp_has_rq(init_attr));
 715 }
 716
 717 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,
 718                                      struct ib_qp_init_attr *init_attr)
 719 {
 720         return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
 721                 hns_roce_qp_has_rq(init_attr));
 722 }
 723
 724 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 725                        struct ib_qp_init_attr *init_attr,
 726                        struct ib_udata *udata,
 727                        struct hns_roce_ib_create_qp *ucmd,
 728                        struct hns_roce_ib_create_qp_resp *resp)
 729 {
 730         struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
 731                 udata, struct hns_roce_ucontext, ibucontext);
 732         struct ib_device *ibdev = &hr_dev->ib_dev;
 733         int ret;
 734
 735         if (udata) {
 736                 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) {
 737                         ret = hns_roce_db_map_user(uctx, udata, ucmd->sdb_addr,
 738                                                    &hr_qp->sdb);
 739                         if (ret) {
 740                                 ibdev_err(ibdev,
 741                                           "Failed to map user SQ doorbell\n");
 742                                 goto err_out;
 743                         }
 744                         hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
 745                         resp->cap_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
 746                 }
 747
 748                 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
 749                         ret = hns_roce_db_map_user(uctx, udata, ucmd->db_addr,
 750                                                    &hr_qp->rdb);
 751                         if (ret) {
 752                                 ibdev_err(ibdev,
 753                                           "Failed to map user RQ doorbell\n");
 754                                 goto err_sdb;
 755                         }
 756                         hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 757                         resp->cap_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 758                 }
 759         } else {
 760                 /* QP doorbell register address */
 761                 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset +
 762                                      DB_REG_OFFSET * hr_dev->priv_uar.index;
 763                 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset +
 764                                      DB_REG_OFFSET * hr_dev->priv_uar.index;
 765
 766                 if (kernel_qp_has_rdb(hr_dev, init_attr)) {
 767                         ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
 768                         if (ret) {
 769                                 ibdev_err(ibdev,
 770                                           "Failed to alloc kernel RQ doorbell\n");
 771                                 goto err_out;
 772                         }
 773                         *hr_qp->rdb.db_record = 0;
 774                         hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 775                 }
 776         }
 777
 778         return 0;
 779 err_sdb:
 780         if (udata && hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
 781                 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
 782 err_out:
 783         return ret;
 784 }
 785
 786 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 787                        struct ib_udata *udata)
 788 {
 789         struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
 790                 udata, struct hns_roce_ucontext, ibucontext);
 791
 792         if (udata) {
 793                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 794                         hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
 795                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
 796                         hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
 797         } else {
 798                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 799                         hns_roce_free_db(hr_dev, &hr_qp->rdb);
 800         }
 801 }
 802
 803 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
 804                              struct hns_roce_qp *hr_qp)
 805 {
 806         struct ib_device *ibdev = &hr_dev->ib_dev;
 807         u64 *sq_wrid = NULL;
 808         u64 *rq_wrid = NULL;
 809         int ret;
 810
 811         sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
 812         if (ZERO_OR_NULL_PTR(sq_wrid)) {
 813                 ibdev_err(ibdev, "Failed to alloc SQ wrid\n");
 814                 return -ENOMEM;
 815         }
 816
 817         if (hr_qp->rq.wqe_cnt) {
 818                 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
 819                 if (ZERO_OR_NULL_PTR(rq_wrid)) {
 820                         ibdev_err(ibdev, "Failed to alloc RQ wrid\n");
 821                         ret = -ENOMEM;
 822                         goto err_sq;
 823                 }
 824         }
 825
 826         hr_qp->sq.wrid = sq_wrid;
 827         hr_qp->rq.wrid = rq_wrid;
 828         return 0;
 829 err_sq:
 830         kfree(sq_wrid);
 831
 832         return ret;
 833 }
 834
 835 static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
 836 {
 837         kfree(hr_qp->rq.wrid);
 838         kfree(hr_qp->sq.wrid);
 839 }
 840
 841 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 842                         struct ib_qp_init_attr *init_attr,
 843                         struct ib_udata *udata,
 844                         struct hns_roce_ib_create_qp *ucmd)
 845 {
 846         struct ib_device *ibdev = &hr_dev->ib_dev;
 847         int ret;
 848
 849         hr_qp->ibqp.qp_type = init_attr->qp_type;
 850
 851         if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
 852                 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
 853         else
 854                 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
 855
 856         ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp,
 857                           hns_roce_qp_has_rq(init_attr));
 858         if (ret) {
 859                 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",
 860                           ret);
 861                 return ret;
 862         }
 863
 864         if (udata) {
 865                 if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd))) {
 866                         ibdev_err(ibdev, "Failed to copy QP ucmd\n");
 867                         return -EFAULT;
 868                 }
 869
 870                 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
 871                 if (ret)
 872                         ibdev_err(ibdev, "Failed to set user SQ size\n");
 873         } else {
 874                 if (init_attr->create_flags &
 875                     IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
 876                         ibdev_err(ibdev, "Failed to check multicast loopback\n");
 877                         return -EINVAL;
 878                 }
 879
 880                 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) {
 881                         ibdev_err(ibdev, "Failed to check ipoib ud lso\n");
 882                         return -EINVAL;
 883                 }
 884
 885                 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
 886                 if (ret)
 887                         ibdev_err(ibdev, "Failed to set kernel SQ size\n");
 888         }
 889
 890         return ret;
 891 }
 892
 893 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 894                                      struct ib_pd *ib_pd,
 895                                      struct ib_qp_init_attr *init_attr,
 896                                      struct ib_udata *udata,
 897                                      struct hns_roce_qp *hr_qp)
 898 {
 899         struct hns_roce_ib_create_qp_resp resp = {};
 900         struct ib_device *ibdev = &hr_dev->ib_dev;
 901         struct hns_roce_ib_create_qp ucmd;
 902         int ret;
 903
 904         mutex_init(&hr_qp->mutex);
 905         spin_lock_init(&hr_qp->sq.lock);
 906         spin_lock_init(&hr_qp->rq.lock);
 907
 908         hr_qp->state = IB_QPS_RESET;
 909         hr_qp->flush_flag = 0;
 910
 911         ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd);
 912         if (ret) {
 913                 ibdev_err(ibdev, "Failed to set QP param\n");
 914                 return ret;
 915         }
 916
 917         if (!udata) {
 918                 ret = alloc_kernel_wrid(hr_dev, hr_qp);
 919                 if (ret) {
 920                         ibdev_err(ibdev, "Failed to alloc wrid\n");
 921                         return ret;
 922                 }
 923         }
 924
 925         ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
 926         if (ret) {
 927                 ibdev_err(ibdev, "Failed to alloc QP doorbell\n");
 928                 goto err_wrid;
 929         }
 930
 931         ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
 932         if (ret) {
 933                 ibdev_err(ibdev, "Failed to alloc QP buffer\n");
 934                 goto err_db;
 935         }
 936
 937         ret = alloc_qpn(hr_dev, hr_qp);
 938         if (ret) {
 939                 ibdev_err(ibdev, "Failed to alloc QPN\n");
 940                 goto err_buf;
 941         }
 942
 943         ret = alloc_qpc(hr_dev, hr_qp);
 944         if (ret) {
 945                 ibdev_err(ibdev, "Failed to alloc QP context\n");
 946                 goto err_qpn;
 947         }
 948
 949         ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
 950         if (ret) {
 951                 ibdev_err(ibdev, "Failed to store QP\n");
 952                 goto err_qpc;
 953         }
 954
 955         if (udata) {
 956                 ret = ib_copy_to_udata(udata, &resp,
 957                                        min(udata->outlen, sizeof(resp)));
 958                 if (ret) {
 959                         ibdev_err(ibdev, "copy qp resp failed!\n");
 960                         goto err_store;
 961                 }
 962         }
 963
 964         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
 965                 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
 966                 if (ret)
 967                         goto err_store;
 968         }
 969
 970         hr_qp->ibqp.qp_num = hr_qp->qpn;
 971         hr_qp->event = hns_roce_ib_qp_event;
 972         atomic_set(&hr_qp->refcount, 1);
 973         init_completion(&hr_qp->free);
 974
 975         return 0;
 976
 977 err_store:
 978         hns_roce_qp_remove(hr_dev, hr_qp);
 979 err_qpc:
 980         free_qpc(hr_dev, hr_qp);
 981 err_qpn:
 982         free_qpn(hr_dev, hr_qp);
 983 err_buf:
 984         free_qp_buf(hr_dev, hr_qp);
 985 err_db:
 986         free_qp_db(hr_dev, hr_qp, udata);
 987 err_wrid:
 988         free_kernel_wrid(hr_qp);
 989         return ret;
 990 }
 991
 992 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 993                          struct ib_udata *udata)
 994 {
 995         if (atomic_dec_and_test(&hr_qp->refcount))
 996                 complete(&hr_qp->free);
 997         wait_for_completion(&hr_qp->free);
 998
 999         free_qpc(hr_dev, hr_qp);
1000         free_qpn(hr_dev, hr_qp);
1001         free_qp_buf(hr_dev, hr_qp);
1002         free_kernel_wrid(hr_qp);
1003         free_qp_db(hr_dev, hr_qp, udata);
1004
1005         kfree(hr_qp);
1006 }
1007
1008 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
1009                                  struct ib_qp_init_attr *init_attr,
1010                                  struct ib_udata *udata)
1011 {
1012         struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1013         struct ib_device *ibdev = &hr_dev->ib_dev;
1014         struct hns_roce_qp *hr_qp;
1015         int ret;
1016
1017         switch (init_attr->qp_type) {
1018         case IB_QPT_RC: {
1019                 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
1020                 if (!hr_qp)
1021                         return ERR_PTR(-ENOMEM);
1022
1023                 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,
1024                                                 hr_qp);
1025                 if (ret) {
1026                         ibdev_err(ibdev, "Create QP 0x%06lx failed(%d)\n",
1027                                   hr_qp->qpn, ret);
1028                         kfree(hr_qp);
1029                         return ERR_PTR(ret);
1030                 }
1031
1032                 break;
1033         }
1034         case IB_QPT_GSI: {
1035                 /* Userspace is not allowed to create special QPs: */
1036                 if (udata) {
1037                         ibdev_err(ibdev, "not support usr space GSI\n");
1038                         return ERR_PTR(-EINVAL);
1039                 }
1040
1041                 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
1042                 if (!hr_qp)
1043                         return ERR_PTR(-ENOMEM);
1044
1045                 hr_qp->port = init_attr->port_num - 1;
1046                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1047
1048                 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,
1049                                                 hr_qp);
1050                 if (ret) {
1051                         ibdev_err(ibdev, "Create GSI QP failed!\n");
1052                         kfree(hr_qp);
1053                         return ERR_PTR(ret);
1054                 }
1055
1056                 break;
1057         }
1058         default:{
1059                 ibdev_err(ibdev, "not support QP type %d\n",
1060                           init_attr->qp_type);
1061                 return ERR_PTR(-EOPNOTSUPP);
1062         }
1063         }
1064
1065         return &hr_qp->ibqp;
1066 }
1067
1068 int to_hr_qp_type(int qp_type)
1069 {
1070         int transport_type;
1071
1072         if (qp_type == IB_QPT_RC)
1073                 transport_type = SERV_TYPE_RC;
1074         else if (qp_type == IB_QPT_UC)
1075                 transport_type = SERV_TYPE_UC;
1076         else if (qp_type == IB_QPT_UD)
1077                 transport_type = SERV_TYPE_UD;
1078         else if (qp_type == IB_QPT_GSI)
1079                 transport_type = SERV_TYPE_UD;
1080         else
1081                 transport_type = -1;
1082
1083         return transport_type;
1084 }
1085
1086 static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1087                               struct hns_roce_qp *hr_qp,
1088                               struct ib_qp_attr *attr, int attr_mask)
1089 {
1090         enum ib_mtu active_mtu;
1091         int p;
1092
1093         p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1094         active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1095
1096         if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1097             attr->path_mtu > hr_dev->caps.max_mtu) ||
1098             attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1099                 ibdev_err(&hr_dev->ib_dev,
1100                         "attr path_mtu(%d)invalid while modify qp",
1101                         attr->path_mtu);
1102                 return -EINVAL;
1103         }
1104
1105         return 0;
1106 }
1107
1108 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1109                                   int attr_mask)
1110 {
1111         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1112         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1113         int p;
1114
1115         if ((attr_mask & IB_QP_PORT) &&
1116             (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1117                 ibdev_err(&hr_dev->ib_dev,
1118                         "attr port_num invalid.attr->port_num=%d\n",
1119                         attr->port_num);
1120                 return -EINVAL;
1121         }
1122
1123         if (attr_mask & IB_QP_PKEY_INDEX) {
1124                 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1125                 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1126                         ibdev_err(&hr_dev->ib_dev,
1127                                 "attr pkey_index invalid.attr->pkey_index=%d\n",
1128                                 attr->pkey_index);
1129                         return -EINVAL;
1130                 }
1131         }
1132
1133         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1134             attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1135                 ibdev_err(&hr_dev->ib_dev,
1136                         "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n",
1137                         attr->max_rd_atomic);
1138                 return -EINVAL;
1139         }
1140
1141         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1142             attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1143                 ibdev_err(&hr_dev->ib_dev,
1144                         "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n",
1145                         attr->max_dest_rd_atomic);
1146                 return -EINVAL;
1147         }
1148
1149         if (attr_mask & IB_QP_PATH_MTU)
1150                 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1151
1152         return 0;
1153 }
1154
1155 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1156                        int attr_mask, struct ib_udata *udata)
1157 {
1158         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1159         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1160         enum ib_qp_state cur_state, new_state;
1161         int ret = -EINVAL;
1162
1163         mutex_lock(&hr_qp->mutex);
1164
1165         cur_state = attr_mask & IB_QP_CUR_STATE ?
1166                     attr->cur_qp_state : (enum ib_qp_state)hr_qp->state;
1167         new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1168
1169         if (ibqp->uobject &&
1170             (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1171                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) {
1172                         hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1173
1174                         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
1175                                 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1176                 } else {
1177                         ibdev_warn(&hr_dev->ib_dev,
1178                                   "flush cqe is not supported in userspace!\n");
1179                         goto out;
1180                 }
1181         }
1182
1183         if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1184                                 attr_mask)) {
1185                 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1186                 goto out;
1187         }
1188
1189         ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1190         if (ret)
1191                 goto out;
1192
1193         if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1194                 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
1195                         ret = -EPERM;
1196                         ibdev_err(&hr_dev->ib_dev,
1197                                   "RST2RST state is not supported\n");
1198                 } else {
1199                         ret = 0;
1200                 }
1201
1202                 goto out;
1203         }
1204
1205         ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1206                                     new_state);
1207
1208 out:
1209         mutex_unlock(&hr_qp->mutex);
1210
1211         return ret;
1212 }
1213
1214 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1215                        __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1216 {
1217         if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1218                 __acquire(&send_cq->lock);
1219                 __acquire(&recv_cq->lock);
1220         } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1221                 spin_lock_irq(&send_cq->lock);
1222                 __acquire(&recv_cq->lock);
1223         } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1224                 spin_lock_irq(&recv_cq->lock);
1225                 __acquire(&send_cq->lock);
1226         } else if (send_cq == recv_cq) {
1227                 spin_lock_irq(&send_cq->lock);
1228                 __acquire(&recv_cq->lock);
1229         } else if (send_cq->cqn < recv_cq->cqn) {
1230                 spin_lock_irq(&send_cq->lock);
1231                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1232         } else {
1233                 spin_lock_irq(&recv_cq->lock);
1234                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1235         }
1236 }
1237
1238 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1239                          struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1240                          __releases(&recv_cq->lock)
1241 {
1242         if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1243                 __release(&recv_cq->lock);
1244                 __release(&send_cq->lock);
1245         } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1246                 __release(&recv_cq->lock);
1247                 spin_unlock(&send_cq->lock);
1248         } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1249                 __release(&send_cq->lock);
1250                 spin_unlock(&recv_cq->lock);
1251         } else if (send_cq == recv_cq) {
1252                 __release(&recv_cq->lock);
1253                 spin_unlock_irq(&send_cq->lock);
1254         } else if (send_cq->cqn < recv_cq->cqn) {
1255                 spin_unlock(&recv_cq->lock);
1256                 spin_unlock_irq(&send_cq->lock);
1257         } else {
1258                 spin_unlock(&send_cq->lock);
1259                 spin_unlock_irq(&recv_cq->lock);
1260         }
1261 }
1262
1263 static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
1264 {
1265         return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
1266 }
1267
1268 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
1269 {
1270         return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1271 }
1272
1273 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n)
1274 {
1275         return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1276 }
1277
1278 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n)
1279 {
1280         return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
1281 }
1282
1283 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
1284                           struct ib_cq *ib_cq)
1285 {
1286         struct hns_roce_cq *hr_cq;
1287         u32 cur;
1288
1289         cur = hr_wq->head - hr_wq->tail;
1290         if (likely(cur + nreq < hr_wq->wqe_cnt))
1291                 return false;
1292
1293         hr_cq = to_hr_cq(ib_cq);
1294         spin_lock(&hr_cq->lock);
1295         cur = hr_wq->head - hr_wq->tail;
1296         spin_unlock(&hr_cq->lock);
1297
1298         return cur + nreq >= hr_wq->wqe_cnt;
1299 }
1300
1301 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1302 {
1303         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1304         int reserved_from_top = 0;
1305         int reserved_from_bot;
1306         int ret;
1307
1308         mutex_init(&qp_table->scc_mutex);
1309         xa_init(&hr_dev->qp_table_xa);
1310
1311         reserved_from_bot = hr_dev->caps.reserved_qps;
1312
1313         ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps,
1314                                    hr_dev->caps.num_qps - 1, reserved_from_bot,
1315                                    reserved_from_top);
1316         if (ret) {
1317                 dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n",
1318                         ret);
1319                 return ret;
1320         }
1321
1322         return 0;
1323 }
1324
1325 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1326 {
1327         hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap);
1328 }