drivers/infiniband/hw/i40iw/i40iw_ctrl.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include "i40iw_osdep.h"
  36 #include "i40iw_register.h"
  37 #include "i40iw_status.h"
  38 #include "i40iw_hmc.h"
  39
  40 #include "i40iw_d.h"
  41 #include "i40iw_type.h"
  42 #include "i40iw_p.h"
  43 #include "i40iw_vf.h"
  44 #include "i40iw_virtchnl.h"
  45
  46 /**
  47  * i40iw_insert_wqe_hdr - write wqe header
  48  * @wqe: cqp wqe for header
  49  * @header: header for the cqp wqe
  50  */
  51 void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
  52 {
  53         wmb();            /* make sure WQE is populated before polarity is set */
  54         set_64bit_val(wqe, 24, header);
  55 }
  56
  57 void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev)
  58 {
  59         if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {
  60                 cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS];
  61                 cqp_timeout->count = 0;
  62         } else {
  63                 if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds)
  64                         cqp_timeout->count++;
  65         }
  66 }
  67
  68 /**
  69  * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
  70  * @cqp: struct for cqp hw
  71  * @val: cqp tail register value
  72  * @tail:wqtail register value
  73  * @error: cqp processing err
  74  */
  75 static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,
  76                                           u32 *val,
  77                                           u32 *tail,
  78                                           u32 *error)
  79 {
  80         if (cqp->dev->is_pf) {
  81                 *val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL);
  82                 *tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL);
  83                 *error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);
  84         } else {
  85                 *val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1);
  86                 *tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL);
  87                 *error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);
  88         }
  89 }
  90
  91 /**
  92  * i40iw_cqp_poll_registers - poll cqp registers
  93  * @cqp: struct for cqp hw
  94  * @tail:wqtail register value
  95  * @count: how many times to try for completion
  96  */
  97 static enum i40iw_status_code i40iw_cqp_poll_registers(
  98                                                 struct i40iw_sc_cqp *cqp,
  99                                                 u32 tail,
 100                                                 u32 count)
 101 {
 102         u32 i = 0;
 103         u32 newtail, error, val;
 104
 105         while (i < count) {
 106                 i++;
 107                 i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error);
 108                 if (error) {
 109                         error = (cqp->dev->is_pf) ?
 110                                  i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
 111                                  i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 112                         return I40IW_ERR_CQP_COMPL_ERROR;
 113                 }
 114                 if (newtail != tail) {
 115                         /* SUCCESS */
 116                         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 117                         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 118                         return 0;
 119                 }
 120                 udelay(I40IW_SLEEP_COUNT);
 121         }
 122         return I40IW_ERR_TIMEOUT;
 123 }
 124
 125 /**
 126  * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
 127  * @buf: ptr to fpm commit buffer
 128  * @info: ptr to i40iw_hmc_obj_info struct
 129  * @sd: number of SDs for HMC objects
 130  *
 131  * parses fpm commit info and copy base value
 132  * of hmc objects in hmc_info
 133  */
 134 static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
 135                                 u64 *buf,
 136                                 struct i40iw_hmc_obj_info *info,
 137                                 u32 *sd)
 138 {
 139         u64 temp;
 140         u64 size;
 141         u64 base = 0;
 142         u32 i, j;
 143         u32 k = 0;
 144
 145         /* copy base values in obj_info */
 146         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
 147                 if ((i == I40IW_HMC_IW_SRQ) ||
 148                         (i == I40IW_HMC_IW_FSIMC) ||
 149                         (i == I40IW_HMC_IW_FSIAV)) {
 150                         info[i].base = 0;
 151                         info[i].cnt = 0;
 152                         continue;
 153                 }
 154                 get_64bit_val(buf, j, &temp);
 155                 info[i].base = RS_64_1(temp, 32) * 512;
 156                 if (info[i].base > base) {
 157                         base = info[i].base;
 158                         k = i;
 159                 }
 160                 if (i == I40IW_HMC_IW_APBVT_ENTRY) {
 161                         info[i].cnt = 1;
 162                         continue;
 163                 }
 164                 if (i == I40IW_HMC_IW_QP)
 165                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 166                 else if (i == I40IW_HMC_IW_CQ)
 167                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 168                 else
 169                         info[i].cnt = (u32)(temp);
 170         }
 171         size = info[k].cnt * info[k].size + info[k].base;
 172         if (size & 0x1FFFFF)
 173                 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
 174         else
 175                 *sd = (u32)(size >> 21);
 176
 177         return 0;
 178 }
 179
 180 /**
 181  * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
 182  * @buf: ptr to fpm query buffer
 183  * @buf_idx: index into buf
 184  * @info: ptr to i40iw_hmc_obj_info struct
 185  * @rsrc_idx: resource index into info
 186  *
 187  * Decode a 64 bit value from fpm query buffer into max count and size
 188  */
 189 static u64 i40iw_sc_decode_fpm_query(u64 *buf,
 190                                             u32 buf_idx,
 191                                             struct i40iw_hmc_obj_info *obj_info,
 192                                             u32 rsrc_idx)
 193 {
 194         u64 temp;
 195         u32 size;
 196
 197         get_64bit_val(buf, buf_idx, &temp);
 198         obj_info[rsrc_idx].max_cnt = (u32)temp;
 199         size = (u32)RS_64_1(temp, 32);
 200         obj_info[rsrc_idx].size = LS_64_1(1, size);
 201
 202         return temp;
 203 }
 204
 205 /**
 206  * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
 207  * @buf: ptr to fpm query buffer
 208  * @info: ptr to i40iw_hmc_obj_info struct
 209  * @hmc_fpm_misc: ptr to fpm data
 210  *
 211  * parses fpm query buffer and copy max_cnt and
 212  * size value of hmc objects in hmc_info
 213  */
 214 static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 215                                 u64 *buf,
 216                                 struct i40iw_hmc_info *hmc_info,
 217                                 struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
 218 {
 219         struct i40iw_hmc_obj_info *obj_info;
 220         u64 temp;
 221         u32 size;
 222         u16 max_pe_sds;
 223
 224         obj_info = hmc_info->hmc_obj;
 225
 226         get_64bit_val(buf, 0, &temp);
 227         hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);
 228         max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);
 229
 230         /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
 231         if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID)
 232                 max_pe_sds--;
 233         hmc_fpm_misc->max_sds = max_pe_sds;
 234         hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
 235
 236         get_64bit_val(buf, 8, &temp);
 237         obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 238         size = (u32)RS_64_1(temp, 32);
 239         obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
 240
 241         get_64bit_val(buf, 16, &temp);
 242         obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 243         size = (u32)RS_64_1(temp, 32);
 244         obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
 245
 246         i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
 247         i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
 248
 249         obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
 250         obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
 251
 252         i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
 253         i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
 254
 255         get_64bit_val(buf, 64, &temp);
 256         obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
 257         obj_info[I40IW_HMC_IW_XFFL].size = 4;
 258         hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
 259         if (!hmc_fpm_misc->xf_block_size)
 260                 return I40IW_ERR_INVALID_SIZE;
 261
 262         i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
 263
 264         get_64bit_val(buf, 80, &temp);
 265         obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
 266         obj_info[I40IW_HMC_IW_Q1FL].size = 4;
 267         hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
 268         if (!hmc_fpm_misc->q1_block_size)
 269                 return I40IW_ERR_INVALID_SIZE;
 270
 271         i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
 272
 273         get_64bit_val(buf, 112, &temp);
 274         obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
 275         obj_info[I40IW_HMC_IW_PBLE].size = 8;
 276
 277         get_64bit_val(buf, 120, &temp);
 278         hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
 279         hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
 280         hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
 281
 282         return 0;
 283 }
 284
 285 /**
 286  * i40iw_fill_qos_list - Change all unknown qs handles to available ones
 287  * @qs_list: list of qs_handles to be fixed with valid qs_handles
 288  */
 289 static void i40iw_fill_qos_list(u16 *qs_list)
 290 {
 291         u16 qshandle = qs_list[0];
 292         int i;
 293
 294         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 295                 if (qs_list[i] == QS_HANDLE_UNKNOWN)
 296                         qs_list[i] = qshandle;
 297                 else
 298                         qshandle = qs_list[i];
 299         }
 300 }
 301
 302 /**
 303  * i40iw_qp_from_entry - Given entry, get to the qp structure
 304  * @entry: Points to list of qp structure
 305  */
 306 static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)
 307 {
 308         if (!entry)
 309                 return NULL;
 310
 311         return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));
 312 }
 313
 314 /**
 315  * i40iw_get_qp - get the next qp from the list given current qp
 316  * @head: Listhead of qp's
 317  * @qp: current qp
 318  */
 319 static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)
 320 {
 321         struct list_head *entry = NULL;
 322         struct list_head *lastentry;
 323
 324         if (list_empty(head))
 325                 return NULL;
 326
 327         if (!qp) {
 328                 entry = head->next;
 329         } else {
 330                 lastentry = &qp->list;
 331                 entry = (lastentry != head) ? lastentry->next : NULL;
 332         }
 333
 334         return i40iw_qp_from_entry(entry);
 335 }
 336
 337 /**
 338  * i40iw_change_l2params - given the new l2 parameters, change all qp
 339  * @vsi: pointer to the vsi structure
 340  * @l2params: New paramaters from l2
 341  */
 342 void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)
 343 {
 344         struct i40iw_sc_dev *dev = vsi->dev;
 345         struct i40iw_sc_qp *qp = NULL;
 346         bool qs_handle_change = false;
 347         unsigned long flags;
 348         u16 qs_handle;
 349         int i;
 350
 351         if (vsi->mtu != l2params->mtu) {
 352                 vsi->mtu = l2params->mtu;
 353                 i40iw_reinitialize_ieq(dev);
 354         }
 355
 356         i40iw_fill_qos_list(l2params->qs_handle_list);
 357         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 358                 qs_handle = l2params->qs_handle_list[i];
 359                 if (vsi->qos[i].qs_handle != qs_handle)
 360                         qs_handle_change = true;
 361                 spin_lock_irqsave(&vsi->qos[i].lock, flags);
 362                 qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 363                 while (qp) {
 364                         if (qs_handle_change) {
 365                                 qp->qs_handle = qs_handle;
 366                                 /* issue cqp suspend command */
 367                                 i40iw_qp_suspend_resume(dev, qp, true);
 368                         }
 369                         qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 370                 }
 371                 spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
 372                 vsi->qos[i].qs_handle = qs_handle;
 373         }
 374 }
 375
 376 /**
 377  * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
 378  * @qp: qp to be removed from qos
 379  */
 380 void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)
 381 {
 382         struct i40iw_sc_vsi *vsi = qp->vsi;
 383         unsigned long flags;
 384
 385         if (!qp->on_qoslist)
 386                 return;
 387         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 388         list_del(&qp->list);
 389         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 390 }
 391
 392 /**
 393  * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
 394  * @qp: qp to be added to qos
 395  */
 396 void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)
 397 {
 398         struct i40iw_sc_vsi *vsi = qp->vsi;
 399         unsigned long flags;
 400
 401         if (qp->on_qoslist)
 402                 return;
 403         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 404         qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
 405         list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
 406         qp->on_qoslist = true;
 407         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 408 }
 409
 410 /**
 411  * i40iw_sc_pd_init - initialize sc pd struct
 412  * @dev: sc device struct
 413  * @pd: sc pd ptr
 414  * @pd_id: pd_id for allocated pd
 415  * @abi_ver: ABI version from user context, -1 if not valid
 416  */
 417 static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev,
 418                              struct i40iw_sc_pd *pd,
 419                              u16 pd_id,
 420                              int abi_ver)
 421 {
 422         pd->size = sizeof(*pd);
 423         pd->pd_id = pd_id;
 424         pd->abi_ver = abi_ver;
 425         pd->dev = dev;
 426 }
 427
 428 /**
 429  * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
 430  * @wqsize: size of the wq (sq, rq, srq) to encoded_size
 431  * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
 432  */
 433 u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
 434 {
 435         u8 encoded_size = 0;
 436
 437         /* cqp sq's hw coded value starts from 1 for size of 4
 438          * while it starts from 0 for qp' wq's.
 439          */
 440         if (cqpsq)
 441                 encoded_size = 1;
 442         wqsize >>= 2;
 443         while (wqsize >>= 1)
 444                 encoded_size++;
 445         return encoded_size;
 446 }
 447
 448 /**
 449  * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
 450  * @cqp: IWARP control queue pair pointer
 451  * @info: IWARP control queue pair init info pointer
 452  *
 453  * Initializes the object and context buffers for a control Queue Pair.
 454  */
 455 static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
 456                                                 struct i40iw_cqp_init_info *info)
 457 {
 458         u8 hw_sq_size;
 459
 460         if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) ||
 461             (info->sq_size < I40IW_CQP_SW_SQSIZE_4) ||
 462             ((info->sq_size & (info->sq_size - 1))))
 463                 return I40IW_ERR_INVALID_SIZE;
 464
 465         hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, true);
 466         cqp->size = sizeof(*cqp);
 467         cqp->sq_size = info->sq_size;
 468         cqp->hw_sq_size = hw_sq_size;
 469         cqp->sq_base = info->sq;
 470         cqp->host_ctx = info->host_ctx;
 471         cqp->sq_pa = info->sq_pa;
 472         cqp->host_ctx_pa = info->host_ctx_pa;
 473         cqp->dev = info->dev;
 474         cqp->struct_ver = info->struct_ver;
 475         cqp->scratch_array = info->scratch_array;
 476         cqp->polarity = 0;
 477         cqp->en_datacenter_tcp = info->en_datacenter_tcp;
 478         cqp->enabled_vf_count = info->enabled_vf_count;
 479         cqp->hmc_profile = info->hmc_profile;
 480         info->dev->cqp = cqp;
 481
 482         I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
 483         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
 484         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
 485         INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);               /* for the cqp commands backlog. */
 486
 487         i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPTAIL, 0);
 488         i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, 0);
 489
 490         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 491                     "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
 492                     __func__, cqp->sq_size, cqp->hw_sq_size,
 493                     cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity);
 494         return 0;
 495 }
 496
 497 /**
 498  * i40iw_sc_cqp_create - create cqp during bringup
 499  * @cqp: struct for cqp hw
 500  * @maj_err: If error, major err number
 501  * @min_err: If error, minor err number
 502  */
 503 static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
 504                                                   u16 *maj_err,
 505                                                   u16 *min_err)
 506 {
 507         u64 temp;
 508         u32 cnt = 0, p1, p2, val = 0, err_code;
 509         enum i40iw_status_code ret_code;
 510
 511         *maj_err = 0;
 512         *min_err = 0;
 513
 514         ret_code = i40iw_allocate_dma_mem(cqp->dev->hw,
 515                                           &cqp->sdbuf,
 516                                           I40IW_UPDATE_SD_BUF_SIZE * cqp->sq_size,
 517                                           I40IW_SD_BUF_ALIGNMENT);
 518
 519         if (ret_code)
 520                 goto exit;
 521
 522         temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
 523                LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
 524
 525         set_64bit_val(cqp->host_ctx, 0, temp);
 526         set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
 527         temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
 528                LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE);
 529         set_64bit_val(cqp->host_ctx, 16, temp);
 530         set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
 531         set_64bit_val(cqp->host_ctx, 32, 0);
 532         set_64bit_val(cqp->host_ctx, 40, 0);
 533         set_64bit_val(cqp->host_ctx, 48, 0);
 534         set_64bit_val(cqp->host_ctx, 56, 0);
 535
 536         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",
 537                         cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8);
 538
 539         p1 = RS_32_1(cqp->host_ctx_pa, 32);
 540         p2 = (u32)cqp->host_ctx_pa;
 541
 542         if (cqp->dev->is_pf) {
 543                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1);
 544                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2);
 545         } else {
 546                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1);
 547                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2);
 548         }
 549         do {
 550                 if (cnt++ > I40IW_DONE_COUNT) {
 551                         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 552                         ret_code = I40IW_ERR_TIMEOUT;
 553                         /*
 554                          * read PFPE_CQPERRORCODES register to get the minor
 555                          * and major error code
 556                          */
 557                         if (cqp->dev->is_pf)
 558                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES);
 559                         else
 560                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 561                         *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);
 562                         *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);
 563                         goto exit;
 564                 }
 565                 udelay(I40IW_SLEEP_COUNT);
 566                 if (cqp->dev->is_pf)
 567                         val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS);
 568                 else
 569                         val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1);
 570         } while (!val);
 571
 572 exit:
 573         if (!ret_code)
 574                 cqp->process_cqp_sds = i40iw_update_sds_noccq;
 575         return ret_code;
 576 }
 577
 578 /**
 579  * i40iw_sc_cqp_post_sq - post of cqp's sq
 580  * @cqp: struct for cqp hw
 581  */
 582 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)
 583 {
 584         if (cqp->dev->is_pf)
 585                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 586         else
 587                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 588
 589         i40iw_debug(cqp->dev,
 590                     I40IW_DEBUG_WQE,
 591                     "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
 592                     __func__,
 593                     cqp->sq_ring.head,
 594                     cqp->sq_ring.tail,
 595                     cqp->sq_ring.size);
 596 }
 597
 598 /**
 599  * i40iw_sc_cqp_get_next_send_wqe_idx - get next WQE on CQP SQ and pass back the index
 600  * @cqp: pointer to CQP structure
 601  * @scratch: private data for CQP WQE
 602  * @wqe_idx: WQE index for next WQE on CQP SQ
 603  */
 604 static u64 *i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp *cqp,
 605                                                u64 scratch, u32 *wqe_idx)
 606 {
 607         u64 *wqe = NULL;
 608         enum i40iw_status_code ret_code;
 609
 610         if (I40IW_RING_FULL_ERR(cqp->sq_ring)) {
 611                 i40iw_debug(cqp->dev,
 612                             I40IW_DEBUG_WQE,
 613                             "%s: ring is full head %x tail %x size %x\n",
 614                             __func__,
 615                             cqp->sq_ring.head,
 616                             cqp->sq_ring.tail,
 617                             cqp->sq_ring.size);
 618                 return NULL;
 619         }
 620         I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
 621         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
 622         if (ret_code)
 623                 return NULL;
 624         if (!*wqe_idx)
 625                 cqp->polarity = !cqp->polarity;
 626
 627         wqe = cqp->sq_base[*wqe_idx].elem;
 628         cqp->scratch_array[*wqe_idx] = scratch;
 629         I40IW_CQP_INIT_WQE(wqe);
 630
 631         return wqe;
 632 }
 633
 634 /**
 635  * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
 636  * @cqp: struct for cqp hw
 637  * @scratch: private data for CQP WQE
 638  */
 639 u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
 640 {
 641         u32 wqe_idx;
 642
 643         return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
 644 }
 645
 646 /**
 647  * i40iw_sc_cqp_destroy - destroy cqp during close
 648  * @cqp: struct for cqp hw
 649  */
 650 static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)
 651 {
 652         u32 cnt = 0, val = 1;
 653         enum i40iw_status_code ret_code = 0;
 654         u32 cqpstat_addr;
 655
 656         if (cqp->dev->is_pf) {
 657                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0);
 658                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0);
 659                 cqpstat_addr = I40E_PFPE_CCQPSTATUS;
 660         } else {
 661                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0);
 662                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0);
 663                 cqpstat_addr = I40E_VFPE_CCQPSTATUS1;
 664         }
 665         do {
 666                 if (cnt++ > I40IW_DONE_COUNT) {
 667                         ret_code = I40IW_ERR_TIMEOUT;
 668                         break;
 669                 }
 670                 udelay(I40IW_SLEEP_COUNT);
 671                 val = i40iw_rd32(cqp->dev->hw, cqpstat_addr);
 672         } while (val);
 673
 674         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 675         return ret_code;
 676 }
 677
 678 /**
 679  * i40iw_sc_ccq_arm - enable intr for control cq
 680  * @ccq: ccq sc struct
 681  */
 682 static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)
 683 {
 684         u64 temp_val;
 685         u16 sw_cq_sel;
 686         u8 arm_next_se;
 687         u8 arm_seq_num;
 688
 689         /* write to cq doorbell shadow area */
 690         /* arm next se should always be zero */
 691         get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
 692
 693         sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
 694         arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
 695
 696         arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
 697         arm_seq_num++;
 698
 699         temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
 700                    LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
 701                    LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
 702                    LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);
 703
 704         set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
 705
 706         wmb();       /* make sure shadow area is updated before arming */
 707
 708         if (ccq->dev->is_pf)
 709                 i40iw_wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id);
 710         else
 711                 i40iw_wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id);
 712 }
 713
 714 /**
 715  * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
 716  * @ccq: ccq sc struct
 717  * @info: completion q entry to return
 718  */
 719 static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(
 720                                         struct i40iw_sc_cq *ccq,
 721                                         struct i40iw_ccq_cqe_info *info)
 722 {
 723         u64 qp_ctx, temp, temp1;
 724         u64 *cqe;
 725         struct i40iw_sc_cqp *cqp;
 726         u32 wqe_idx;
 727         u8 polarity;
 728         enum i40iw_status_code ret_code = 0;
 729
 730         if (ccq->cq_uk.avoid_mem_cflct)
 731                 cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk);
 732         else
 733                 cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk);
 734
 735         get_64bit_val(cqe, 24, &temp);
 736         polarity = (u8)RS_64(temp, I40IW_CQ_VALID);
 737         if (polarity != ccq->cq_uk.polarity)
 738                 return I40IW_ERR_QUEUE_EMPTY;
 739
 740         get_64bit_val(cqe, 8, &qp_ctx);
 741         cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx;
 742         info->error = (bool)RS_64(temp, I40IW_CQ_ERROR);
 743         info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 744         if (info->error) {
 745                 info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);
 746                 info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 747         }
 748         wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);
 749         info->scratch = cqp->scratch_array[wqe_idx];
 750
 751         get_64bit_val(cqe, 16, &temp1);
 752         info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);
 753         get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
 754         info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);
 755         info->cqp = cqp;
 756
 757         /*  move the head for cq */
 758         I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
 759         if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
 760                 ccq->cq_uk.polarity ^= 1;
 761
 762         /* update cq tail in cq shadow memory also */
 763         I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
 764         set_64bit_val(ccq->cq_uk.shadow_area,
 765                       0,
 766                       I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
 767         wmb(); /* write shadow area before tail */
 768         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 769         ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 770
 771         return ret_code;
 772 }
 773
 774 /**
 775  * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
 776  * @cqp: struct for cqp hw
 777  * @op_code: cqp opcode for completion
 778  * @info: completion q entry to return
 779  */
 780 static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done(
 781                                         struct i40iw_sc_cqp *cqp,
 782                                         u8 op_code,
 783                                         struct i40iw_ccq_cqe_info *compl_info)
 784 {
 785         struct i40iw_ccq_cqe_info info;
 786         struct i40iw_sc_cq *ccq;
 787         enum i40iw_status_code ret_code = 0;
 788         u32 cnt = 0;
 789
 790         memset(&info, 0, sizeof(info));
 791         ccq = cqp->dev->ccq;
 792         while (1) {
 793                 if (cnt++ > I40IW_DONE_COUNT)
 794                         return I40IW_ERR_TIMEOUT;
 795
 796                 if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) {
 797                         udelay(I40IW_SLEEP_COUNT);
 798                         continue;
 799                 }
 800
 801                 if (info.error) {
 802                         ret_code = I40IW_ERR_CQP_COMPL_ERROR;
 803                         break;
 804                 }
 805                 /* check if opcode is cq create */
 806                 if (op_code != info.op_code) {
 807                         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 808                                     "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
 809                                     __func__, op_code, info.op_code);
 810                 }
 811                 /* success, exit out of the loop */
 812                 if (op_code == info.op_code)
 813                         break;
 814         }
 815
 816         if (compl_info)
 817                 memcpy(compl_info, &info, sizeof(*compl_info));
 818
 819         return ret_code;
 820 }
 821
 822 /**
 823  * i40iw_sc_manage_push_page - Handle push page
 824  * @cqp: struct for cqp hw
 825  * @info: push page info
 826  * @scratch: u64 saved to be used during cqp completion
 827  * @post_sq: flag for cqp db to ring
 828  */
 829 static enum i40iw_status_code i40iw_sc_manage_push_page(
 830                                 struct i40iw_sc_cqp *cqp,
 831                                 struct i40iw_cqp_manage_push_page_info *info,
 832                                 u64 scratch,
 833                                 bool post_sq)
 834 {
 835         u64 *wqe;
 836         u64 header;
 837
 838         if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
 839                 return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
 840
 841         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 842         if (!wqe)
 843                 return I40IW_ERR_RING_FULL;
 844
 845         set_64bit_val(wqe, 16, info->qs_handle);
 846
 847         header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) |
 848                  LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |
 849                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
 850                  LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
 851
 852         i40iw_insert_wqe_hdr(wqe, header);
 853
 854         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
 855                         wqe, I40IW_CQP_WQE_SIZE * 8);
 856
 857         if (post_sq)
 858                 i40iw_sc_cqp_post_sq(cqp);
 859         return 0;
 860 }
 861
 862 /**
 863  * i40iw_sc_manage_hmc_pm_func_table - manage of function table
 864  * @cqp: struct for cqp hw
 865  * @scratch: u64 saved to be used during cqp completion
 866  * @vf_index: vf index for cqp
 867  * @free_pm_fcn: function number
 868  * @post_sq: flag for cqp db to ring
 869  */
 870 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table(
 871                                 struct i40iw_sc_cqp *cqp,
 872                                 u64 scratch,
 873                                 u8 vf_index,
 874                                 bool free_pm_fcn,
 875                                 bool post_sq)
 876 {
 877         u64 *wqe;
 878         u64 header;
 879
 880         if (vf_index >= I40IW_MAX_VF_PER_PF)
 881                 return I40IW_ERR_INVALID_VF_ID;
 882         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 883         if (!wqe)
 884                 return I40IW_ERR_RING_FULL;
 885
 886         header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) |
 887                  LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) |
 888                  LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) |
 889                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 890
 891         i40iw_insert_wqe_hdr(wqe, header);
 892         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 893                         wqe, I40IW_CQP_WQE_SIZE * 8);
 894         if (post_sq)
 895                 i40iw_sc_cqp_post_sq(cqp);
 896         return 0;
 897 }
 898
 899 /**
 900  * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
 901  * @cqp: struct for cqp hw
 902  * @scratch: u64 saved to be used during cqp completion
 903  * @hmc_profile_type: type of profile to set
 904  * @vf_num: vf number for profile
 905  * @post_sq: flag for cqp db to ring
 906  * @poll_registers: flag to poll register for cqp completion
 907  */
 908 static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile(
 909                                 struct i40iw_sc_cqp *cqp,
 910                                 u64 scratch,
 911                                 u8 hmc_profile_type,
 912                                 u8 vf_num, bool post_sq,
 913                                 bool poll_registers)
 914 {
 915         u64 *wqe;
 916         u64 header;
 917         u32 val, tail, error;
 918         enum i40iw_status_code ret_code = 0;
 919
 920         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 921         if (!wqe)
 922                 return I40IW_ERR_RING_FULL;
 923
 924         set_64bit_val(wqe, 16,
 925                       (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) |
 926                                 LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM)));
 927
 928         header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) |
 929                        LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 930
 931         i40iw_insert_wqe_hdr(wqe, header);
 932
 933         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 934                         wqe, I40IW_CQP_WQE_SIZE * 8);
 935
 936         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
 937         if (error)
 938                 return I40IW_ERR_CQP_COMPL_ERROR;
 939
 940         if (post_sq) {
 941                 i40iw_sc_cqp_post_sq(cqp);
 942                 if (poll_registers)
 943                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000);
 944                 else
 945                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
 946                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
 947                                                                  NULL);
 948         }
 949
 950         return ret_code;
 951 }
 952
 953 /**
 954  * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
 955  * @cqp: struct for cqp hw
 956  */
 957 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)
 958 {
 959         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);
 960 }
 961
 962 /**
 963  * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
 964  * @cqp: struct for cqp hw
 965  */
 966 static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)
 967 {
 968         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);
 969 }
 970
 971 /**
 972  * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
 973  * @cqp: struct for cqp hw
 974  * @scratch: u64 saved to be used during cqp completion
 975  * @hmc_fn_id: hmc function id
 976  * @commit_fpm_mem; Memory for fpm values
 977  * @post_sq: flag for cqp db to ring
 978  * @wait_type: poll ccq or cqp registers for cqp completion
 979  */
 980 static enum i40iw_status_code i40iw_sc_commit_fpm_values(
 981                                         struct i40iw_sc_cqp *cqp,
 982                                         u64 scratch,
 983                                         u8 hmc_fn_id,
 984                                         struct i40iw_dma_mem *commit_fpm_mem,
 985                                         bool post_sq,
 986                                         u8 wait_type)
 987 {
 988         u64 *wqe;
 989         u64 header;
 990         u32 tail, val, error;
 991         enum i40iw_status_code ret_code = 0;
 992
 993         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 994         if (!wqe)
 995                 return I40IW_ERR_RING_FULL;
 996
 997         set_64bit_val(wqe, 16, hmc_fn_id);
 998         set_64bit_val(wqe, 32, commit_fpm_mem->pa);
 999
1000         header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1001                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1002
1003         i40iw_insert_wqe_hdr(wqe, header);
1004
1005         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",
1006                         wqe, I40IW_CQP_WQE_SIZE * 8);
1007
1008         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1009         if (error)
1010                 return I40IW_ERR_CQP_COMPL_ERROR;
1011
1012         if (post_sq) {
1013                 i40iw_sc_cqp_post_sq(cqp);
1014
1015                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1016                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1017                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1018                         ret_code = i40iw_sc_commit_fpm_values_done(cqp);
1019         }
1020
1021         return ret_code;
1022 }
1023
1024 /**
1025  * i40iw_sc_query_rdma_features_done - poll cqp for query features done
1026  * @cqp: struct for cqp hw
1027  */
1028 static enum i40iw_status_code
1029 i40iw_sc_query_rdma_features_done(struct i40iw_sc_cqp *cqp)
1030 {
1031         return i40iw_sc_poll_for_cqp_op_done(
1032                 cqp, I40IW_CQP_OP_QUERY_RDMA_FEATURES, NULL);
1033 }
1034
1035 /**
1036  * i40iw_sc_query_rdma_features - query rdma features
1037  * @cqp: struct for cqp hw
1038  * @feat_mem: holds PA for HW to use
1039  * @scratch: u64 saved to be used during cqp completion
1040  */
1041 static enum i40iw_status_code
1042 i40iw_sc_query_rdma_features(struct i40iw_sc_cqp *cqp,
1043                              struct i40iw_dma_mem *feat_mem, u64 scratch)
1044 {
1045         u64 *wqe;
1046         u64 header;
1047
1048         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1049         if (!wqe)
1050                 return I40IW_ERR_RING_FULL;
1051
1052         set_64bit_val(wqe, 32, feat_mem->pa);
1053
1054         header = LS_64(I40IW_CQP_OP_QUERY_RDMA_FEATURES, I40IW_CQPSQ_OPCODE) |
1055                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | feat_mem->size;
1056
1057         i40iw_insert_wqe_hdr(wqe, header);
1058
1059         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY RDMA FEATURES WQE",
1060                         wqe, I40IW_CQP_WQE_SIZE * 8);
1061
1062         i40iw_sc_cqp_post_sq(cqp);
1063
1064         return 0;
1065 }
1066
1067 /**
1068  * i40iw_get_rdma_features - get RDMA features
1069  * @dev - sc device struct
1070  */
1071 enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev)
1072 {
1073         enum i40iw_status_code ret_code;
1074         struct i40iw_dma_mem feat_buf;
1075         u64 temp;
1076         u16 byte_idx, feat_type, feat_cnt;
1077
1078         ret_code = i40iw_allocate_dma_mem(dev->hw, &feat_buf,
1079                                           I40IW_FEATURE_BUF_SIZE,
1080                                           I40IW_FEATURE_BUF_ALIGNMENT);
1081
1082         if (ret_code)
1083                 return I40IW_ERR_NO_MEMORY;
1084
1085         ret_code = i40iw_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
1086         if (!ret_code)
1087                 ret_code = i40iw_sc_query_rdma_features_done(dev->cqp);
1088
1089         if (ret_code)
1090                 goto exit;
1091
1092         get_64bit_val(feat_buf.va, 0, &temp);
1093         feat_cnt = RS_64(temp, I40IW_FEATURE_CNT);
1094         if (feat_cnt < I40IW_MAX_FEATURES) {
1095                 ret_code = I40IW_ERR_INVALID_FEAT_CNT;
1096                 goto exit;
1097         } else if (feat_cnt > I40IW_MAX_FEATURES) {
1098                 i40iw_debug(dev, I40IW_DEBUG_CQP,
1099                             "features buf size insufficient\n");
1100         }
1101
1102         for (byte_idx = 0, feat_type = 0; feat_type < I40IW_MAX_FEATURES;
1103              feat_type++, byte_idx += 8) {
1104                 get_64bit_val((u64 *)feat_buf.va, byte_idx, &temp);
1105                 dev->feature_info[feat_type] = RS_64(temp, I40IW_FEATURE_INFO);
1106         }
1107 exit:
1108         i40iw_free_dma_mem(dev->hw, &feat_buf);
1109
1110         return ret_code;
1111 }
1112
1113 /**
1114  * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
1115  * @cqp: struct for cqp hw
1116  */
1117 static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)
1118 {
1119         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);
1120 }
1121
1122 /**
1123  * i40iw_sc_query_fpm_values - cqp wqe query fpm values
1124  * @cqp: struct for cqp hw
1125  * @scratch: u64 saved to be used during cqp completion
1126  * @hmc_fn_id: hmc function id
1127  * @query_fpm_mem: memory for return fpm values
1128  * @post_sq: flag for cqp db to ring
1129  * @wait_type: poll ccq or cqp registers for cqp completion
1130  */
1131 static enum i40iw_status_code i40iw_sc_query_fpm_values(
1132                                         struct i40iw_sc_cqp *cqp,
1133                                         u64 scratch,
1134                                         u8 hmc_fn_id,
1135                                         struct i40iw_dma_mem *query_fpm_mem,
1136                                         bool post_sq,
1137                                         u8 wait_type)
1138 {
1139         u64 *wqe;
1140         u64 header;
1141         u32 tail, val, error;
1142         enum i40iw_status_code ret_code = 0;
1143
1144         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1145         if (!wqe)
1146                 return I40IW_ERR_RING_FULL;
1147
1148         set_64bit_val(wqe, 16, hmc_fn_id);
1149         set_64bit_val(wqe, 32, query_fpm_mem->pa);
1150
1151         header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1152                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1153
1154         i40iw_insert_wqe_hdr(wqe, header);
1155
1156         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",
1157                         wqe, I40IW_CQP_WQE_SIZE * 8);
1158
1159         /* read the tail from CQP_TAIL register */
1160         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1161
1162         if (error)
1163                 return I40IW_ERR_CQP_COMPL_ERROR;
1164
1165         if (post_sq) {
1166                 i40iw_sc_cqp_post_sq(cqp);
1167                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1168                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1169                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1170                         ret_code = i40iw_sc_query_fpm_values_done(cqp);
1171         }
1172
1173         return ret_code;
1174 }
1175
1176 /**
1177  * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
1178  * @cqp: struct for cqp hw
1179  * @info: arp entry information
1180  * @scratch: u64 saved to be used during cqp completion
1181  * @post_sq: flag for cqp db to ring
1182  */
1183 static enum i40iw_status_code i40iw_sc_add_arp_cache_entry(
1184                                 struct i40iw_sc_cqp *cqp,
1185                                 struct i40iw_add_arp_cache_entry_info *info,
1186                                 u64 scratch,
1187                                 bool post_sq)
1188 {
1189         u64 *wqe;
1190         u64 temp, header;
1191
1192         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1193         if (!wqe)
1194                 return I40IW_ERR_RING_FULL;
1195         set_64bit_val(wqe, 8, info->reach_max);
1196
1197         temp = info->mac_addr[5] |
1198                LS_64_1(info->mac_addr[4], 8) |
1199                LS_64_1(info->mac_addr[3], 16) |
1200                LS_64_1(info->mac_addr[2], 24) |
1201                LS_64_1(info->mac_addr[1], 32) |
1202                LS_64_1(info->mac_addr[0], 40);
1203
1204         set_64bit_val(wqe, 16, temp);
1205
1206         header = info->arp_index |
1207                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1208                  LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) |
1209                  LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) |
1210                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1211
1212         i40iw_insert_wqe_hdr(wqe, header);
1213
1214         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
1215                         wqe, I40IW_CQP_WQE_SIZE * 8);
1216
1217         if (post_sq)
1218                 i40iw_sc_cqp_post_sq(cqp);
1219         return 0;
1220 }
1221
1222 /**
1223  * i40iw_sc_del_arp_cache_entry - dele arp cache entry
1224  * @cqp: struct for cqp hw
1225  * @scratch: u64 saved to be used during cqp completion
1226  * @arp_index: arp index to delete arp entry
1227  * @post_sq: flag for cqp db to ring
1228  */
1229 static enum i40iw_status_code i40iw_sc_del_arp_cache_entry(
1230                                         struct i40iw_sc_cqp *cqp,
1231                                         u64 scratch,
1232                                         u16 arp_index,
1233                                         bool post_sq)
1234 {
1235         u64 *wqe;
1236         u64 header;
1237
1238         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1239         if (!wqe)
1240                 return I40IW_ERR_RING_FULL;
1241
1242         header = arp_index |
1243                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1244                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1245         i40iw_insert_wqe_hdr(wqe, header);
1246
1247         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
1248                         wqe, I40IW_CQP_WQE_SIZE * 8);
1249
1250         if (post_sq)
1251                 i40iw_sc_cqp_post_sq(cqp);
1252         return 0;
1253 }
1254
1255 /**
1256  * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
1257  * @cqp: struct for cqp hw
1258  * @scratch: u64 saved to be used during cqp completion
1259  * @arp_index: arp index to delete arp entry
1260  * @post_sq: flag for cqp db to ring
1261  */
1262 static enum i40iw_status_code i40iw_sc_query_arp_cache_entry(
1263                                 struct i40iw_sc_cqp *cqp,
1264                                 u64 scratch,
1265                                 u16 arp_index,
1266                                 bool post_sq)
1267 {
1268         u64 *wqe;
1269         u64 header;
1270
1271         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1272         if (!wqe)
1273                 return I40IW_ERR_RING_FULL;
1274
1275         header = arp_index |
1276                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1277                  LS_64(1, I40IW_CQPSQ_MAT_QUERY) |
1278                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1279
1280         i40iw_insert_wqe_hdr(wqe, header);
1281
1282         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
1283                         wqe, I40IW_CQP_WQE_SIZE * 8);
1284
1285         if (post_sq)
1286                 i40iw_sc_cqp_post_sq(cqp);
1287         return 0;
1288 }
1289
1290 /**
1291  * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
1292  * @cqp: struct for cqp hw
1293  * @info: info for apbvt entry to add or delete
1294  * @scratch: u64 saved to be used during cqp completion
1295  * @post_sq: flag for cqp db to ring
1296  */
1297 static enum i40iw_status_code i40iw_sc_manage_apbvt_entry(
1298                                 struct i40iw_sc_cqp *cqp,
1299                                 struct i40iw_apbvt_info *info,
1300                                 u64 scratch,
1301                                 bool post_sq)
1302 {
1303         u64 *wqe;
1304         u64 header;
1305
1306         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1307         if (!wqe)
1308                 return I40IW_ERR_RING_FULL;
1309
1310         set_64bit_val(wqe, 16, info->port);
1311
1312         header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) |
1313                  LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) |
1314                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1315
1316         i40iw_insert_wqe_hdr(wqe, header);
1317
1318         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",
1319                         wqe, I40IW_CQP_WQE_SIZE * 8);
1320
1321         if (post_sq)
1322                 i40iw_sc_cqp_post_sq(cqp);
1323         return 0;
1324 }
1325
1326 /**
1327  * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
1328  * @cqp: struct for cqp hw
1329  * @info: info for quad hash to manage
1330  * @scratch: u64 saved to be used during cqp completion
1331  * @post_sq: flag for cqp db to ring
1332  *
1333  * This is called before connection establishment is started. For passive connections, when
1334  * listener is created, it will call with entry type of  I40IW_QHASH_TYPE_TCP_SYN with local
1335  * ip address and tcp port. When SYN is received (passive connections) or
1336  * sent (active connections), this routine is called with entry type of
1337  * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
1338  *
1339  * When iwarp connection is done and its state moves to RTS, the quad hash entry in
1340  * the hardware will point to iwarp's qp number and requires no calls from the driver.
1341  */
1342 static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
1343                                         struct i40iw_sc_cqp *cqp,
1344                                         struct i40iw_qhash_table_info *info,
1345                                         u64 scratch,
1346                                         bool post_sq)
1347 {
1348         u64 *wqe;
1349         u64 qw1 = 0;
1350         u64 qw2 = 0;
1351         u64 temp;
1352         struct i40iw_sc_vsi *vsi = info->vsi;
1353
1354         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1355         if (!wqe)
1356                 return I40IW_ERR_RING_FULL;
1357
1358         temp = info->mac_addr[5] |
1359                 LS_64_1(info->mac_addr[4], 8) |
1360                 LS_64_1(info->mac_addr[3], 16) |
1361                 LS_64_1(info->mac_addr[2], 24) |
1362                 LS_64_1(info->mac_addr[1], 32) |
1363                 LS_64_1(info->mac_addr[0], 40);
1364
1365         set_64bit_val(wqe, 0, temp);
1366
1367         qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) |
1368               LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);
1369         if (info->ipv4_valid) {
1370                 set_64bit_val(wqe,
1371                               48,
1372                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1373         } else {
1374                 set_64bit_val(wqe,
1375                               56,
1376                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1377                               LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1378
1379                 set_64bit_val(wqe,
1380                               48,
1381                               LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1382                               LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1383         }
1384         qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
1385         if (info->vlan_valid)
1386                 qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
1387         set_64bit_val(wqe, 16, qw2);
1388         if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
1389                 qw1 |= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT);
1390                 if (!info->ipv4_valid) {
1391                         set_64bit_val(wqe,
1392                                       40,
1393                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1394                                       LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1395                         set_64bit_val(wqe,
1396                                       32,
1397                                       LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1398                                       LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1399                 } else {
1400                         set_64bit_val(wqe,
1401                                       32,
1402                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1403                 }
1404         }
1405
1406         set_64bit_val(wqe, 8, qw1);
1407         temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) |
1408                LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) |
1409                LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) |
1410                LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) |
1411                LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) |
1412                LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);
1413
1414         i40iw_insert_wqe_hdr(wqe, temp);
1415
1416         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",
1417                         wqe, I40IW_CQP_WQE_SIZE * 8);
1418
1419         if (post_sq)
1420                 i40iw_sc_cqp_post_sq(cqp);
1421         return 0;
1422 }
1423
1424 /**
1425  * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
1426  * @cqp: struct for cqp hw
1427  * @scratch: u64 saved to be used during cqp completion
1428  * @post_sq: flag for cqp db to ring
1429  */
1430 static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry(
1431                                         struct i40iw_sc_cqp *cqp,
1432                                         u64 scratch,
1433                                         bool post_sq)
1434 {
1435         u64 *wqe;
1436         u64 header;
1437
1438         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1439         if (!wqe)
1440                 return I40IW_ERR_RING_FULL;
1441         header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) |
1442                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1443
1444         i40iw_insert_wqe_hdr(wqe, header);
1445         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
1446                         wqe, I40IW_CQP_WQE_SIZE * 8);
1447         if (post_sq)
1448                 i40iw_sc_cqp_post_sq(cqp);
1449         return 0;
1450 }
1451
1452 /**
1453  * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
1454  * @cqp: struct for cqp hw
1455  * @info:mac addr info
1456  * @scratch: u64 saved to be used during cqp completion
1457  * @post_sq: flag for cqp db to ring
1458  */
1459 static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry(
1460                                 struct i40iw_sc_cqp *cqp,
1461                                 struct i40iw_local_mac_ipaddr_entry_info *info,
1462                                 u64 scratch,
1463                                 bool post_sq)
1464 {
1465         u64 *wqe;
1466         u64 temp, header;
1467
1468         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1469         if (!wqe)
1470                 return I40IW_ERR_RING_FULL;
1471         temp = info->mac_addr[5] |
1472                 LS_64_1(info->mac_addr[4], 8) |
1473                 LS_64_1(info->mac_addr[3], 16) |
1474                 LS_64_1(info->mac_addr[2], 24) |
1475                 LS_64_1(info->mac_addr[1], 32) |
1476                 LS_64_1(info->mac_addr[0], 40);
1477
1478         set_64bit_val(wqe, 32, temp);
1479
1480         header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1481                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1482                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1483
1484         i40iw_insert_wqe_hdr(wqe, header);
1485
1486         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",
1487                         wqe, I40IW_CQP_WQE_SIZE * 8);
1488
1489         if (post_sq)
1490                 i40iw_sc_cqp_post_sq(cqp);
1491         return 0;
1492 }
1493
1494 /**
1495  * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
1496  * @cqp: struct for cqp hw
1497  * @scratch: u64 saved to be used during cqp completion
1498  * @entry_idx: index of mac entry
1499  * @ ignore_ref_count: to force mac adde delete
1500  * @post_sq: flag for cqp db to ring
1501  */
1502 static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry(
1503                                 struct i40iw_sc_cqp *cqp,
1504                                 u64 scratch,
1505                                 u8 entry_idx,
1506                                 u8 ignore_ref_count,
1507                                 bool post_sq)
1508 {
1509         u64 *wqe;
1510         u64 header;
1511
1512         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1513         if (!wqe)
1514                 return I40IW_ERR_RING_FULL;
1515         header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1516                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1517                  LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) |
1518                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
1519                  LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);
1520
1521         i40iw_insert_wqe_hdr(wqe, header);
1522
1523         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
1524                         wqe, I40IW_CQP_WQE_SIZE * 8);
1525
1526         if (post_sq)
1527                 i40iw_sc_cqp_post_sq(cqp);
1528         return 0;
1529 }
1530
1531 /**
1532  * i40iw_sc_cqp_nop - send a nop wqe
1533  * @cqp: struct for cqp hw
1534  * @scratch: u64 saved to be used during cqp completion
1535  * @post_sq: flag for cqp db to ring
1536  */
1537 static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,
1538                                                u64 scratch,
1539                                                bool post_sq)
1540 {
1541         u64 *wqe;
1542         u64 header;
1543
1544         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1545         if (!wqe)
1546                 return I40IW_ERR_RING_FULL;
1547         header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) |
1548                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1549         i40iw_insert_wqe_hdr(wqe, header);
1550         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE",
1551                         wqe, I40IW_CQP_WQE_SIZE * 8);
1552
1553         if (post_sq)
1554                 i40iw_sc_cqp_post_sq(cqp);
1555         return 0;
1556 }
1557
1558 /**
1559  * i40iw_sc_ceq_init - initialize ceq
1560  * @ceq: ceq sc structure
1561  * @info: ceq initialization info
1562  */
1563 static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,
1564                                                 struct i40iw_ceq_init_info *info)
1565 {
1566         u32 pble_obj_cnt;
1567
1568         if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) ||
1569             (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES))
1570                 return I40IW_ERR_INVALID_SIZE;
1571
1572         if (info->ceq_id >= I40IW_MAX_CEQID)
1573                 return I40IW_ERR_INVALID_CEQ_ID;
1574
1575         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1576
1577         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1578                 return I40IW_ERR_INVALID_PBLE_INDEX;
1579
1580         ceq->size = sizeof(*ceq);
1581         ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base;
1582         ceq->ceq_id = info->ceq_id;
1583         ceq->dev = info->dev;
1584         ceq->elem_cnt = info->elem_cnt;
1585         ceq->ceq_elem_pa = info->ceqe_pa;
1586         ceq->virtual_map = info->virtual_map;
1587
1588         ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
1589         ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
1590         ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
1591
1592         ceq->tph_en = info->tph_en;
1593         ceq->tph_val = info->tph_val;
1594         ceq->polarity = 1;
1595         I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
1596         ceq->dev->ceq[info->ceq_id] = ceq;
1597
1598         return 0;
1599 }
1600
1601 /**
1602  * i40iw_sc_ceq_create - create ceq wqe
1603  * @ceq: ceq sc structure
1604  * @scratch: u64 saved to be used during cqp completion
1605  * @post_sq: flag for cqp db to ring
1606  */
1607 static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,
1608                                                   u64 scratch,
1609                                                   bool post_sq)
1610 {
1611         struct i40iw_sc_cqp *cqp;
1612         u64 *wqe;
1613         u64 header;
1614
1615         cqp = ceq->dev->cqp;
1616         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1617         if (!wqe)
1618                 return I40IW_ERR_RING_FULL;
1619         set_64bit_val(wqe, 16, ceq->elem_cnt);
1620         set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
1621         set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
1622         set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL));
1623
1624         header = ceq->ceq_id |
1625                  LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) |
1626                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1627                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1628                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1629                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1630
1631         i40iw_insert_wqe_hdr(wqe, header);
1632
1633         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",
1634                         wqe, I40IW_CQP_WQE_SIZE * 8);
1635
1636         if (post_sq)
1637                 i40iw_sc_cqp_post_sq(cqp);
1638         return 0;
1639 }
1640
1641 /**
1642  * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
1643  * @ceq: ceq sc structure
1644  */
1645 static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)
1646 {
1647         struct i40iw_sc_cqp *cqp;
1648
1649         cqp = ceq->dev->cqp;
1650         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);
1651 }
1652
1653 /**
1654  * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
1655  * @ceq: ceq sc structure
1656  */
1657 static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)
1658 {
1659         struct i40iw_sc_cqp *cqp;
1660
1661         cqp = ceq->dev->cqp;
1662         cqp->process_cqp_sds = i40iw_update_sds_noccq;
1663         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);
1664 }
1665
1666 /**
1667  * i40iw_sc_cceq_create - create cceq
1668  * @ceq: ceq sc structure
1669  * @scratch: u64 saved to be used during cqp completion
1670  */
1671 static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)
1672 {
1673         enum i40iw_status_code ret_code;
1674
1675         ret_code = i40iw_sc_ceq_create(ceq, scratch, true);
1676         if (!ret_code)
1677                 ret_code = i40iw_sc_cceq_create_done(ceq);
1678         return ret_code;
1679 }
1680
1681 /**
1682  * i40iw_sc_ceq_destroy - destroy ceq
1683  * @ceq: ceq sc structure
1684  * @scratch: u64 saved to be used during cqp completion
1685  * @post_sq: flag for cqp db to ring
1686  */
1687 static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,
1688                                                    u64 scratch,
1689                                                    bool post_sq)
1690 {
1691         struct i40iw_sc_cqp *cqp;
1692         u64 *wqe;
1693         u64 header;
1694
1695         cqp = ceq->dev->cqp;
1696         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1697         if (!wqe)
1698                 return I40IW_ERR_RING_FULL;
1699         set_64bit_val(wqe, 16, ceq->elem_cnt);
1700         set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
1701         header = ceq->ceq_id |
1702                  LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) |
1703                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1704                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1705                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1706                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1707         i40iw_insert_wqe_hdr(wqe, header);
1708         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",
1709                         wqe, I40IW_CQP_WQE_SIZE * 8);
1710
1711         if (post_sq)
1712                 i40iw_sc_cqp_post_sq(cqp);
1713         return 0;
1714 }
1715
1716 /**
1717  * i40iw_sc_process_ceq - process ceq
1718  * @dev: sc device struct
1719  * @ceq: ceq sc structure
1720  */
1721 static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq)
1722 {
1723         u64 temp;
1724         u64 *ceqe;
1725         struct i40iw_sc_cq *cq = NULL;
1726         u8 polarity;
1727
1728         ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);
1729         get_64bit_val(ceqe, 0, &temp);
1730         polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);
1731         if (polarity != ceq->polarity)
1732                 return cq;
1733
1734         cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1);
1735
1736         I40IW_RING_MOVE_TAIL(ceq->ceq_ring);
1737         if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0)
1738                 ceq->polarity ^= 1;
1739
1740         if (dev->is_pf)
1741                 i40iw_wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id);
1742         else
1743                 i40iw_wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id);
1744
1745         return cq;
1746 }
1747
1748 /**
1749  * i40iw_sc_aeq_init - initialize aeq
1750  * @aeq: aeq structure ptr
1751  * @info: aeq initialization info
1752  */
1753 static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,
1754                                                 struct i40iw_aeq_init_info *info)
1755 {
1756         u32 pble_obj_cnt;
1757
1758         if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) ||
1759             (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES))
1760                 return I40IW_ERR_INVALID_SIZE;
1761         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1762
1763         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1764                 return I40IW_ERR_INVALID_PBLE_INDEX;
1765
1766         aeq->size = sizeof(*aeq);
1767         aeq->polarity = 1;
1768         aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base;
1769         aeq->dev = info->dev;
1770         aeq->elem_cnt = info->elem_cnt;
1771
1772         aeq->aeq_elem_pa = info->aeq_elem_pa;
1773         I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
1774         info->dev->aeq = aeq;
1775
1776         aeq->virtual_map = info->virtual_map;
1777         aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
1778         aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
1779         aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
1780         info->dev->aeq = aeq;
1781         return 0;
1782 }
1783
1784 /**
1785  * i40iw_sc_aeq_create - create aeq
1786  * @aeq: aeq structure ptr
1787  * @scratch: u64 saved to be used during cqp completion
1788  * @post_sq: flag for cqp db to ring
1789  */
1790 static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,
1791                                                   u64 scratch,
1792                                                   bool post_sq)
1793 {
1794         u64 *wqe;
1795         struct i40iw_sc_cqp *cqp;
1796         u64 header;
1797
1798         cqp = aeq->dev->cqp;
1799         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1800         if (!wqe)
1801                 return I40IW_ERR_RING_FULL;
1802         set_64bit_val(wqe, 16, aeq->elem_cnt);
1803         set_64bit_val(wqe, 32,
1804                       (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
1805         set_64bit_val(wqe, 48,
1806                       (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
1807
1808         header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) |
1809                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1810                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1811                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1812
1813         i40iw_insert_wqe_hdr(wqe, header);
1814         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",
1815                         wqe, I40IW_CQP_WQE_SIZE * 8);
1816         if (post_sq)
1817                 i40iw_sc_cqp_post_sq(cqp);
1818         return 0;
1819 }
1820
1821 /**
1822  * i40iw_sc_aeq_destroy - destroy aeq during close
1823  * @aeq: aeq structure ptr
1824  * @scratch: u64 saved to be used during cqp completion
1825  * @post_sq: flag for cqp db to ring
1826  */
1827 static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,
1828                                                    u64 scratch,
1829                                                    bool post_sq)
1830 {
1831         u64 *wqe;
1832         struct i40iw_sc_cqp *cqp;
1833         u64 header;
1834
1835         cqp = aeq->dev->cqp;
1836         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1837         if (!wqe)
1838                 return I40IW_ERR_RING_FULL;
1839         set_64bit_val(wqe, 16, aeq->elem_cnt);
1840         set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
1841         header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) |
1842                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1843                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1844                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1845         i40iw_insert_wqe_hdr(wqe, header);
1846
1847         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",
1848                         wqe, I40IW_CQP_WQE_SIZE * 8);
1849         if (post_sq)
1850                 i40iw_sc_cqp_post_sq(cqp);
1851         return 0;
1852 }
1853
1854 /**
1855  * i40iw_sc_get_next_aeqe - get next aeq entry
1856  * @aeq: aeq structure ptr
1857  * @info: aeqe info to be returned
1858  */
1859 static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
1860                                                      struct i40iw_aeqe_info *info)
1861 {
1862         u64 temp, compl_ctx;
1863         u64 *aeqe;
1864         u16 wqe_idx;
1865         u8 ae_src;
1866         u8 polarity;
1867
1868         aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);
1869         get_64bit_val(aeqe, 0, &compl_ctx);
1870         get_64bit_val(aeqe, 8, &temp);
1871         polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);
1872
1873         if (aeq->polarity != polarity)
1874                 return I40IW_ERR_QUEUE_EMPTY;
1875
1876         i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);
1877
1878         ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);
1879         wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);
1880         info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);
1881         info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);
1882         info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);
1883         info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);
1884         info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);
1885         info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);
1886
1887         switch (info->ae_id) {
1888         case I40IW_AE_PRIV_OPERATION_DENIED:
1889         case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
1890         case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
1891         case I40IW_AE_BAD_CLOSE:
1892         case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:
1893         case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO:
1894         case I40IW_AE_STAG_ZERO_INVALID:
1895         case I40IW_AE_IB_RREQ_AND_Q1_FULL:
1896         case I40IW_AE_WQE_UNEXPECTED_OPCODE:
1897         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
1898         case I40IW_AE_DDP_UBE_INVALID_MO:
1899         case I40IW_AE_DDP_UBE_INVALID_QN:
1900         case I40IW_AE_DDP_NO_L_BIT:
1901         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
1902         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
1903         case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
1904         case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
1905         case I40IW_AE_INVALID_ARP_ENTRY:
1906         case I40IW_AE_INVALID_TCP_OPTION_RCVD:
1907         case I40IW_AE_STALE_ARP_ENTRY:
1908         case I40IW_AE_LLP_CLOSE_COMPLETE:
1909         case I40IW_AE_LLP_CONNECTION_RESET:
1910         case I40IW_AE_LLP_FIN_RECEIVED:
1911         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
1912         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
1913         case I40IW_AE_LLP_SYN_RECEIVED:
1914         case I40IW_AE_LLP_TERMINATE_RECEIVED:
1915         case I40IW_AE_LLP_TOO_MANY_RETRIES:
1916         case I40IW_AE_LLP_DOUBT_REACHABILITY:
1917         case I40IW_AE_RESET_SENT:
1918         case I40IW_AE_TERMINATE_SENT:
1919         case I40IW_AE_RESET_NOT_SENT:
1920         case I40IW_AE_LCE_QP_CATASTROPHIC:
1921         case I40IW_AE_QP_SUSPEND_COMPLETE:
1922                 info->qp = true;
1923                 info->compl_ctx = compl_ctx;
1924                 ae_src = I40IW_AE_SOURCE_RSVD;
1925                 break;
1926         case I40IW_AE_LCE_CQ_CATASTROPHIC:
1927                 info->cq = true;
1928                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1929                 ae_src = I40IW_AE_SOURCE_RSVD;
1930                 break;
1931         }
1932
1933         switch (ae_src) {
1934         case I40IW_AE_SOURCE_RQ:
1935         case I40IW_AE_SOURCE_RQ_0011:
1936                 info->qp = true;
1937                 info->wqe_idx = wqe_idx;
1938                 info->compl_ctx = compl_ctx;
1939                 break;
1940         case I40IW_AE_SOURCE_CQ:
1941         case I40IW_AE_SOURCE_CQ_0110:
1942         case I40IW_AE_SOURCE_CQ_1010:
1943         case I40IW_AE_SOURCE_CQ_1110:
1944                 info->cq = true;
1945                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1946                 break;
1947         case I40IW_AE_SOURCE_SQ:
1948         case I40IW_AE_SOURCE_SQ_0111:
1949                 info->qp = true;
1950                 info->sq = true;
1951                 info->wqe_idx = wqe_idx;
1952                 info->compl_ctx = compl_ctx;
1953                 break;
1954         case I40IW_AE_SOURCE_IN_RR_WR:
1955         case I40IW_AE_SOURCE_IN_RR_WR_1011:
1956                 info->qp = true;
1957                 info->compl_ctx = compl_ctx;
1958                 info->in_rdrsp_wr = true;
1959                 break;
1960         case I40IW_AE_SOURCE_OUT_RR:
1961         case I40IW_AE_SOURCE_OUT_RR_1111:
1962                 info->qp = true;
1963                 info->compl_ctx = compl_ctx;
1964                 info->out_rdrsp = true;
1965                 break;
1966         case I40IW_AE_SOURCE_RSVD:
1967                 /* fallthrough */
1968         default:
1969                 break;
1970         }
1971         I40IW_RING_MOVE_TAIL(aeq->aeq_ring);
1972         if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0)
1973                 aeq->polarity ^= 1;
1974         return 0;
1975 }
1976
1977 /**
1978  * i40iw_sc_repost_aeq_entries - repost completed aeq entries
1979  * @dev: sc device struct
1980  * @count: allocate count
1981  */
1982 static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
1983                                                           u32 count)
1984 {
1985
1986         if (dev->is_pf)
1987                 i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
1988         else
1989                 i40iw_wr32(dev->hw, I40E_VFPE_AEQALLOC1, count);
1990
1991         return 0;
1992 }
1993
1994 /**
1995  * i40iw_sc_aeq_create_done - create aeq
1996  * @aeq: aeq structure ptr
1997  */
1998 static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)
1999 {
2000         struct i40iw_sc_cqp *cqp;
2001
2002         cqp = aeq->dev->cqp;
2003         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);
2004 }
2005
2006 /**
2007  * i40iw_sc_aeq_destroy_done - destroy of aeq during close
2008  * @aeq: aeq structure ptr
2009  */
2010 static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)
2011 {
2012         struct i40iw_sc_cqp *cqp;
2013
2014         cqp = aeq->dev->cqp;
2015         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);
2016 }
2017
2018 /**
2019  * i40iw_sc_ccq_init - initialize control cq
2020  * @cq: sc's cq ctruct
2021  * @info: info for control cq initialization
2022  */
2023 static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,
2024                                                 struct i40iw_ccq_init_info *info)
2025 {
2026         u32 pble_obj_cnt;
2027
2028         if (info->num_elem < I40IW_MIN_CQ_SIZE || info->num_elem > I40IW_MAX_CQ_SIZE)
2029                 return I40IW_ERR_INVALID_SIZE;
2030
2031         if (info->ceq_id > I40IW_MAX_CEQID)
2032                 return I40IW_ERR_INVALID_CEQ_ID;
2033
2034         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2035
2036         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
2037                 return I40IW_ERR_INVALID_PBLE_INDEX;
2038
2039         cq->cq_pa = info->cq_pa;
2040         cq->cq_uk.cq_base = info->cq_base;
2041         cq->shadow_area_pa = info->shadow_area_pa;
2042         cq->cq_uk.shadow_area = info->shadow_area;
2043         cq->shadow_read_threshold = info->shadow_read_threshold;
2044         cq->dev = info->dev;
2045         cq->ceq_id = info->ceq_id;
2046         cq->cq_uk.cq_size = info->num_elem;
2047         cq->cq_type = I40IW_CQ_TYPE_CQP;
2048         cq->ceqe_mask = info->ceqe_mask;
2049         I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
2050
2051         cq->cq_uk.cq_id = 0;    /* control cq is id 0 always */
2052         cq->ceq_id_valid = info->ceq_id_valid;
2053         cq->tph_en = info->tph_en;
2054         cq->tph_val = info->tph_val;
2055         cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
2056
2057         cq->pbl_list = info->pbl_list;
2058         cq->virtual_map = info->virtual_map;
2059         cq->pbl_chunk_size = info->pbl_chunk_size;
2060         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2061         cq->cq_uk.polarity = true;
2062
2063         /* following are only for iw cqs so initialize them to zero */
2064         cq->cq_uk.cqe_alloc_reg = NULL;
2065         info->dev->ccq = cq;
2066         return 0;
2067 }
2068
2069 /**
2070  * i40iw_sc_ccq_create_done - poll cqp for ccq create
2071  * @ccq: ccq sc struct
2072  */
2073 static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)
2074 {
2075         struct i40iw_sc_cqp *cqp;
2076
2077         cqp = ccq->dev->cqp;
2078         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);
2079 }
2080
2081 /**
2082  * i40iw_sc_ccq_create - create control cq
2083  * @ccq: ccq sc struct
2084  * @scratch: u64 saved to be used during cqp completion
2085  * @check_overflow: overlow flag for ccq
2086  * @post_sq: flag for cqp db to ring
2087  */
2088 static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,
2089                                                   u64 scratch,
2090                                                   bool check_overflow,
2091                                                   bool post_sq)
2092 {
2093         u64 *wqe;
2094         struct i40iw_sc_cqp *cqp;
2095         u64 header;
2096         enum i40iw_status_code ret_code;
2097
2098         cqp = ccq->dev->cqp;
2099         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2100         if (!wqe)
2101                 return I40IW_ERR_RING_FULL;
2102         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
2103         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
2104         set_64bit_val(wqe, 16,
2105                       LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2106         set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa));
2107         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
2108         set_64bit_val(wqe, 48,
2109                       (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0));
2110         set_64bit_val(wqe, 56,
2111                       LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL));
2112
2113         header = ccq->cq_uk.cq_id |
2114                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2115                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2116                  LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2117                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2118                  LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2119                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2120                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2121                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2122                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2123                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2124
2125         i40iw_insert_wqe_hdr(wqe, header);
2126
2127         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",
2128                         wqe, I40IW_CQP_WQE_SIZE * 8);
2129
2130         if (post_sq) {
2131                 i40iw_sc_cqp_post_sq(cqp);
2132                 ret_code = i40iw_sc_ccq_create_done(ccq);
2133                 if (ret_code)
2134                         return ret_code;
2135         }
2136         cqp->process_cqp_sds = i40iw_cqp_sds_cmd;
2137
2138         return 0;
2139 }
2140
2141 /**
2142  * i40iw_sc_ccq_destroy - destroy ccq during close
2143  * @ccq: ccq sc struct
2144  * @scratch: u64 saved to be used during cqp completion
2145  * @post_sq: flag for cqp db to ring
2146  */
2147 static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,
2148                                                    u64 scratch,
2149                                                    bool post_sq)
2150 {
2151         struct i40iw_sc_cqp *cqp;
2152         u64 *wqe;
2153         u64 header;
2154         enum i40iw_status_code ret_code = 0;
2155         u32 tail, val, error;
2156
2157         cqp = ccq->dev->cqp;
2158         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2159         if (!wqe)
2160                 return I40IW_ERR_RING_FULL;
2161         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
2162         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
2163         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
2164
2165         header = ccq->cq_uk.cq_id |
2166                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2167                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2168                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2169                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2170                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2171                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2172                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2173
2174         i40iw_insert_wqe_hdr(wqe, header);
2175
2176         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",
2177                         wqe, I40IW_CQP_WQE_SIZE * 8);
2178
2179         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
2180         if (error)
2181                 return I40IW_ERR_CQP_COMPL_ERROR;
2182
2183         if (post_sq) {
2184                 i40iw_sc_cqp_post_sq(cqp);
2185                 ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
2186         }
2187
2188         cqp->process_cqp_sds = i40iw_update_sds_noccq;
2189
2190         return ret_code;
2191 }
2192
2193 /**
2194  * i40iw_sc_cq_init - initialize completion q
2195  * @cq: cq struct
2196  * @info: cq initialization info
2197  */
2198 static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq,
2199                                                struct i40iw_cq_init_info *info)
2200 {
2201         u32 __iomem *cqe_alloc_reg = NULL;
2202         enum i40iw_status_code ret_code;
2203         u32 pble_obj_cnt;
2204         u32 arm_offset;
2205
2206         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2207
2208         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
2209                 return I40IW_ERR_INVALID_PBLE_INDEX;
2210
2211         cq->cq_pa = info->cq_base_pa;
2212         cq->dev = info->dev;
2213         cq->ceq_id = info->ceq_id;
2214         arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;
2215         if (i40iw_get_hw_addr(cq->dev))
2216                 cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq->dev) +
2217                                               arm_offset);
2218         info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;
2219         ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
2220         if (ret_code)
2221                 return ret_code;
2222         cq->virtual_map = info->virtual_map;
2223         cq->pbl_chunk_size = info->pbl_chunk_size;
2224         cq->ceqe_mask = info->ceqe_mask;
2225         cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP;
2226
2227         cq->shadow_area_pa = info->shadow_area_pa;
2228         cq->shadow_read_threshold = info->shadow_read_threshold;
2229
2230         cq->ceq_id_valid = info->ceq_id_valid;
2231         cq->tph_en = info->tph_en;
2232         cq->tph_val = info->tph_val;
2233
2234         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2235
2236         return 0;
2237 }
2238
2239 /**
2240  * i40iw_sc_cq_create - create completion q
2241  * @cq: cq struct
2242  * @scratch: u64 saved to be used during cqp completion
2243  * @check_overflow: flag for overflow check
2244  * @post_sq: flag for cqp db to ring
2245  */
2246 static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq,
2247                                                  u64 scratch,
2248                                                  bool check_overflow,
2249                                                  bool post_sq)
2250 {
2251         u64 *wqe;
2252         struct i40iw_sc_cqp *cqp;
2253         u64 header;
2254
2255         if (cq->cq_uk.cq_id > I40IW_MAX_CQID)
2256                 return I40IW_ERR_INVALID_CQ_ID;
2257
2258         if (cq->ceq_id > I40IW_MAX_CEQID)
2259                 return I40IW_ERR_INVALID_CEQ_ID;
2260
2261         cqp = cq->dev->cqp;
2262         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2263         if (!wqe)
2264                 return I40IW_ERR_RING_FULL;
2265
2266         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2267         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2268         set_64bit_val(wqe,
2269                       16,
2270                       LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2271
2272         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2273
2274         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2275         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2276         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2277
2278         header = cq->cq_uk.cq_id |
2279                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2280                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2281                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2282                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2283                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2284                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2285                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2286                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2287                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2288                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2289
2290         i40iw_insert_wqe_hdr(wqe, header);
2291
2292         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",
2293                         wqe, I40IW_CQP_WQE_SIZE * 8);
2294
2295         if (post_sq)
2296                 i40iw_sc_cqp_post_sq(cqp);
2297         return 0;
2298 }
2299
2300 /**
2301  * i40iw_sc_cq_destroy - destroy completion q
2302  * @cq: cq struct
2303  * @scratch: u64 saved to be used during cqp completion
2304  * @post_sq: flag for cqp db to ring
2305  */
2306 static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,
2307                                                   u64 scratch,
2308                                                   bool post_sq)
2309 {
2310         struct i40iw_sc_cqp *cqp;
2311         u64 *wqe;
2312         u64 header;
2313
2314         cqp = cq->dev->cqp;
2315         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2316         if (!wqe)
2317                 return I40IW_ERR_RING_FULL;
2318         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2319         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2320         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2321         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2322
2323         header = cq->cq_uk.cq_id |
2324                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2325                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2326                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2327                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2328                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2329                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2330                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2331                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2332                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2333
2334         i40iw_insert_wqe_hdr(wqe, header);
2335
2336         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",
2337                         wqe, I40IW_CQP_WQE_SIZE * 8);
2338
2339         if (post_sq)
2340                 i40iw_sc_cqp_post_sq(cqp);
2341         return 0;
2342 }
2343
2344 /**
2345  * i40iw_sc_cq_modify - modify a Completion Queue
2346  * @cq: cq struct
2347  * @info: modification info struct
2348  * @scratch:
2349  * @post_sq: flag to post to sq
2350  */
2351 static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,
2352                                                  struct i40iw_modify_cq_info *info,
2353                                                  u64 scratch,
2354                                                  bool post_sq)
2355 {
2356         struct i40iw_sc_cqp *cqp;
2357         u64 *wqe;
2358         u64 header;
2359         u32 cq_size, ceq_id, first_pm_pbl_idx;
2360         u8 pbl_chunk_size;
2361         bool virtual_map, ceq_id_valid, check_overflow;
2362         u32 pble_obj_cnt;
2363
2364         if (info->ceq_valid && (info->ceq_id > I40IW_MAX_CEQID))
2365                 return I40IW_ERR_INVALID_CEQ_ID;
2366
2367         pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2368
2369         if (info->cq_resize && info->virtual_map &&
2370             (info->first_pm_pbl_idx >= pble_obj_cnt))
2371                 return I40IW_ERR_INVALID_PBLE_INDEX;
2372
2373         cqp = cq->dev->cqp;
2374         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2375         if (!wqe)
2376                 return I40IW_ERR_RING_FULL;
2377
2378         cq->pbl_list = info->pbl_list;
2379         cq->cq_pa = info->cq_pa;
2380         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2381
2382         cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
2383         if (info->ceq_change) {
2384                 ceq_id_valid = true;
2385                 ceq_id = info->ceq_id;
2386         } else {
2387                 ceq_id_valid = cq->ceq_id_valid;
2388                 ceq_id = ceq_id_valid ? cq->ceq_id : 0;
2389         }
2390         virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
2391         first_pm_pbl_idx = (info->cq_resize ?
2392                             (info->virtual_map ? info->first_pm_pbl_idx : 0) :
2393                             (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2394         pbl_chunk_size = (info->cq_resize ?
2395                           (info->virtual_map ? info->pbl_chunk_size : 0) :
2396                           (cq->virtual_map ? cq->pbl_chunk_size : 0));
2397         check_overflow = info->check_overflow_change ? info->check_overflow :
2398                          cq->check_overflow;
2399         cq->cq_uk.cq_size = cq_size;
2400         cq->ceq_id_valid = ceq_id_valid;
2401         cq->ceq_id = ceq_id;
2402         cq->virtual_map = virtual_map;
2403         cq->first_pm_pbl_idx = first_pm_pbl_idx;
2404         cq->pbl_chunk_size = pbl_chunk_size;
2405         cq->check_overflow = check_overflow;
2406
2407         set_64bit_val(wqe, 0, cq_size);
2408         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2409         set_64bit_val(wqe, 16,
2410                       LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2411         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2412         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2413         set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0));
2414         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2415
2416         header = cq->cq_uk.cq_id |
2417                  LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) |
2418                  LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) |
2419                  LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) |
2420                  LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2421                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2422                  LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2423                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2424                  LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2425                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2426                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2427                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2428
2429         i40iw_insert_wqe_hdr(wqe, header);
2430
2431         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",
2432                         wqe, I40IW_CQP_WQE_SIZE * 8);
2433
2434         if (post_sq)
2435                 i40iw_sc_cqp_post_sq(cqp);
2436         return 0;
2437 }
2438
2439 /**
2440  * i40iw_sc_qp_init - initialize qp
2441  * @qp: sc qp
2442  * @info: initialization qp info
2443  */
2444 static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
2445                                                struct i40iw_qp_init_info *info)
2446 {
2447         u32 __iomem *wqe_alloc_reg = NULL;
2448         enum i40iw_status_code ret_code;
2449         u32 pble_obj_cnt;
2450         u8 wqe_size;
2451         u32 offset;
2452
2453         qp->dev = info->pd->dev;
2454         qp->vsi = info->vsi;
2455         qp->sq_pa = info->sq_pa;
2456         qp->rq_pa = info->rq_pa;
2457         qp->hw_host_ctx_pa = info->host_ctx_pa;
2458         qp->q2_pa = info->q2_pa;
2459         qp->shadow_area_pa = info->shadow_area_pa;
2460
2461         qp->q2_buf = info->q2;
2462         qp->pd = info->pd;
2463         qp->hw_host_ctx = info->host_ctx;
2464         offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;
2465         if (i40iw_get_hw_addr(qp->pd->dev))
2466                 wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
2467                                               offset);
2468
2469         info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;
2470         info->qp_uk_init_info.abi_ver = qp->pd->abi_ver;
2471         ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
2472         if (ret_code)
2473                 return ret_code;
2474         qp->virtual_map = info->virtual_map;
2475
2476         pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2477
2478         if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) ||
2479             (info->virtual_map && (info->rq_pa >= pble_obj_cnt)))
2480                 return I40IW_ERR_INVALID_PBLE_INDEX;
2481
2482         qp->llp_stream_handle = (void *)(-1);
2483         qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP;
2484
2485         qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
2486                                                     false);
2487         i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
2488                     __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size);
2489
2490         switch (qp->pd->abi_ver) {
2491         case 4:
2492                 ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
2493                                                        &wqe_size);
2494                 if (ret_code)
2495                         return ret_code;
2496                 break;
2497         case 5: /* fallthrough until next ABI version */
2498         default:
2499                 if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
2500                         return I40IW_ERR_INVALID_FRAG_COUNT;
2501                 wqe_size = I40IW_MAX_WQE_SIZE_RQ;
2502                 break;
2503         }
2504         qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size *
2505                                 (wqe_size / I40IW_QP_WQE_MIN_SIZE), false);
2506         i40iw_debug(qp->dev, I40IW_DEBUG_WQE,
2507                     "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
2508                     __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
2509         qp->sq_tph_val = info->sq_tph_val;
2510         qp->rq_tph_val = info->rq_tph_val;
2511         qp->sq_tph_en = info->sq_tph_en;
2512         qp->rq_tph_en = info->rq_tph_en;
2513         qp->rcv_tph_en = info->rcv_tph_en;
2514         qp->xmit_tph_en = info->xmit_tph_en;
2515         qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
2516
2517         return 0;
2518 }
2519
2520 /**
2521  * i40iw_sc_qp_create - create qp
2522  * @qp: sc qp
2523  * @info: qp create info
2524  * @scratch: u64 saved to be used during cqp completion
2525  * @post_sq: flag for cqp db to ring
2526  */
2527 static enum i40iw_status_code i40iw_sc_qp_create(
2528                                 struct i40iw_sc_qp *qp,
2529                                 struct i40iw_create_qp_info *info,
2530                                 u64 scratch,
2531                                 bool post_sq)
2532 {
2533         struct i40iw_sc_cqp *cqp;
2534         u64 *wqe;
2535         u64 header;
2536
2537         if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) ||
2538             (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID))
2539                 return I40IW_ERR_INVALID_QP_ID;
2540
2541         cqp = qp->pd->dev->cqp;
2542         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2543         if (!wqe)
2544                 return I40IW_ERR_RING_FULL;
2545
2546         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2547
2548         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2549
2550         header = qp->qp_uk.qp_id |
2551                  LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |
2552                  LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) |
2553                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2554                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2555                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2556                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2557                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2558                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2559                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2560
2561         i40iw_insert_wqe_hdr(wqe, header);
2562         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",
2563                         wqe, I40IW_CQP_WQE_SIZE * 8);
2564
2565         if (post_sq)
2566                 i40iw_sc_cqp_post_sq(cqp);
2567         return 0;
2568 }
2569
2570 /**
2571  * i40iw_sc_qp_modify - modify qp cqp wqe
2572  * @qp: sc qp
2573  * @info: modify qp info
2574  * @scratch: u64 saved to be used during cqp completion
2575  * @post_sq: flag for cqp db to ring
2576  */
2577 static enum i40iw_status_code i40iw_sc_qp_modify(
2578                                 struct i40iw_sc_qp *qp,
2579                                 struct i40iw_modify_qp_info *info,
2580                                 u64 scratch,
2581                                 bool post_sq)
2582 {
2583         u64 *wqe;
2584         struct i40iw_sc_cqp *cqp;
2585         u64 header;
2586         u8 term_actions = 0;
2587         u8 term_len = 0;
2588
2589         cqp = qp->pd->dev->cqp;
2590         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2591         if (!wqe)
2592                 return I40IW_ERR_RING_FULL;
2593         if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) {
2594                 if (info->dont_send_fin)
2595                         term_actions += I40IWQP_TERM_SEND_TERM_ONLY;
2596                 if (info->dont_send_term)
2597                         term_actions += I40IWQP_TERM_SEND_FIN_ONLY;
2598                 if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) ||
2599                     (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))
2600                         term_len = info->termlen;
2601         }
2602
2603         set_64bit_val(wqe,
2604                       8,
2605                       LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));
2606
2607         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2608         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2609
2610         header = qp->qp_uk.qp_id |
2611                  LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) |
2612                  LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) |
2613                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2614                  LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) |
2615                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2616                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2617                  LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) |
2618                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2619                  LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2620                  LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) |
2621                  LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) |
2622                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2623                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2624                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2625
2626         i40iw_insert_wqe_hdr(wqe, header);
2627
2628         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",
2629                         wqe, I40IW_CQP_WQE_SIZE * 8);
2630
2631         if (post_sq)
2632                 i40iw_sc_cqp_post_sq(cqp);
2633         return 0;
2634 }
2635
2636 /**
2637  * i40iw_sc_qp_destroy - cqp destroy qp
2638  * @qp: sc qp
2639  * @scratch: u64 saved to be used during cqp completion
2640  * @remove_hash_idx: flag if to remove hash idx
2641  * @ignore_mw_bnd: memory window bind flag
2642  * @post_sq: flag for cqp db to ring
2643  */
2644 static enum i40iw_status_code i40iw_sc_qp_destroy(
2645                                         struct i40iw_sc_qp *qp,
2646                                         u64 scratch,
2647                                         bool remove_hash_idx,
2648                                         bool ignore_mw_bnd,
2649                                         bool post_sq)
2650 {
2651         u64 *wqe;
2652         struct i40iw_sc_cqp *cqp;
2653         u64 header;
2654
2655         i40iw_qp_rem_qos(qp);
2656         cqp = qp->pd->dev->cqp;
2657         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2658         if (!wqe)
2659                 return I40IW_ERR_RING_FULL;
2660         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2661         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2662
2663         header = qp->qp_uk.qp_id |
2664                  LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) |
2665                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2666                  LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) |
2667                  LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2668                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2669
2670         i40iw_insert_wqe_hdr(wqe, header);
2671         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",
2672                         wqe, I40IW_CQP_WQE_SIZE * 8);
2673
2674         if (post_sq)
2675                 i40iw_sc_cqp_post_sq(cqp);
2676         return 0;
2677 }
2678
2679 /**
2680  * i40iw_sc_qp_flush_wqes - flush qp's wqe
2681  * @qp: sc qp
2682  * @info: dlush information
2683  * @scratch: u64 saved to be used during cqp completion
2684  * @post_sq: flag for cqp db to ring
2685  */
2686 static enum i40iw_status_code i40iw_sc_qp_flush_wqes(
2687                                 struct i40iw_sc_qp *qp,
2688                                 struct i40iw_qp_flush_info *info,
2689                                 u64 scratch,
2690                                 bool post_sq)
2691 {
2692         u64 temp = 0;
2693         u64 *wqe;
2694         struct i40iw_sc_cqp *cqp;
2695         u64 header;
2696         bool flush_sq = false, flush_rq = false;
2697
2698         if (info->rq && !qp->flush_rq)
2699                 flush_rq = true;
2700
2701         if (info->sq && !qp->flush_sq)
2702                 flush_sq = true;
2703
2704         qp->flush_sq |= flush_sq;
2705         qp->flush_rq |= flush_rq;
2706         if (!flush_sq && !flush_rq)
2707                 return 0;
2708
2709         cqp = qp->pd->dev->cqp;
2710         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2711         if (!wqe)
2712                 return I40IW_ERR_RING_FULL;
2713         if (info->userflushcode) {
2714                 if (flush_rq) {
2715                         temp |= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) |
2716                                 LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);
2717                 }
2718                 if (flush_sq) {
2719                         temp |= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) |
2720                                 LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);
2721                 }
2722         }
2723         set_64bit_val(wqe, 16, temp);
2724
2725         temp = (info->generate_ae) ?
2726                 info->ae_code | LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;
2727
2728         set_64bit_val(wqe, 8, temp);
2729
2730         header = qp->qp_uk.qp_id |
2731                  LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) |
2732                  LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2733                  LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) |
2734                  LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) |
2735                  LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) |
2736                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2737
2738         i40iw_insert_wqe_hdr(wqe, header);
2739
2740         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",
2741                         wqe, I40IW_CQP_WQE_SIZE * 8);
2742
2743         if (post_sq)
2744                 i40iw_sc_cqp_post_sq(cqp);
2745         return 0;
2746 }
2747
2748 /**
2749  * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP
2750  * @qp: sc qp
2751  * @info: gen ae information
2752  * @scratch: u64 saved to be used during cqp completion
2753  * @post_sq: flag for cqp db to ring
2754  */
2755 static enum i40iw_status_code i40iw_sc_gen_ae(
2756                                 struct i40iw_sc_qp *qp,
2757                                 struct i40iw_gen_ae_info *info,
2758                                 u64 scratch,
2759                                 bool post_sq)
2760 {
2761         u64 temp;
2762         u64 *wqe;
2763         struct i40iw_sc_cqp *cqp;
2764         u64 header;
2765
2766         cqp = qp->pd->dev->cqp;
2767         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2768         if (!wqe)
2769                 return I40IW_ERR_RING_FULL;
2770
2771         temp = info->ae_code |
2772                LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE);
2773
2774         set_64bit_val(wqe, 8, temp);
2775
2776         header = qp->qp_uk.qp_id |
2777                  LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) |
2778                  LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2779                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2780
2781         i40iw_insert_wqe_hdr(wqe, header);
2782
2783         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "GEN_AE WQE",
2784                         wqe, I40IW_CQP_WQE_SIZE * 8);
2785
2786         if (post_sq)
2787                 i40iw_sc_cqp_post_sq(cqp);
2788         return 0;
2789 }
2790
2791 /**
2792  * i40iw_sc_qp_upload_context - upload qp's context
2793  * @dev: sc device struct
2794  * @info: upload context info ptr for return
2795  * @scratch: u64 saved to be used during cqp completion
2796  * @post_sq: flag for cqp db to ring
2797  */
2798 static enum i40iw_status_code i40iw_sc_qp_upload_context(
2799                                         struct i40iw_sc_dev *dev,
2800                                         struct i40iw_upload_context_info *info,
2801                                         u64 scratch,
2802                                         bool post_sq)
2803 {
2804         u64 *wqe;
2805         struct i40iw_sc_cqp *cqp;
2806         u64 header;
2807
2808         cqp = dev->cqp;
2809         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2810         if (!wqe)
2811                 return I40IW_ERR_RING_FULL;
2812         set_64bit_val(wqe, 16, info->buf_pa);
2813
2814         header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) |
2815                  LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) |
2816                  LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) |
2817                  LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) |
2818                  LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) |
2819                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2820
2821         i40iw_insert_wqe_hdr(wqe, header);
2822
2823         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
2824                         wqe, I40IW_CQP_WQE_SIZE * 8);
2825
2826         if (post_sq)
2827                 i40iw_sc_cqp_post_sq(cqp);
2828         return 0;
2829 }
2830
2831 /**
2832  * i40iw_sc_qp_setctx - set qp's context
2833  * @qp: sc qp
2834  * @qp_ctx: context ptr
2835  * @info: ctx info
2836  */
2837 static enum i40iw_status_code i40iw_sc_qp_setctx(
2838                                 struct i40iw_sc_qp *qp,
2839                                 u64 *qp_ctx,
2840                                 struct i40iw_qp_host_ctx_info *info)
2841 {
2842         struct i40iwarp_offload_info *iw;
2843         struct i40iw_tcp_offload_info *tcp;
2844         struct i40iw_sc_vsi *vsi;
2845         struct i40iw_sc_dev *dev;
2846         u64 qw0, qw3, qw7 = 0;
2847
2848         iw = info->iwarp_info;
2849         tcp = info->tcp_info;
2850         vsi = qp->vsi;
2851         dev = qp->dev;
2852         if (info->add_to_qoslist) {
2853                 qp->user_pri = info->user_pri;
2854                 i40iw_qp_add_qos(qp);
2855                 i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",
2856                             __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
2857         }
2858         qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |
2859               LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |
2860               LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
2861               LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) |
2862               LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) |
2863               LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) |
2864               LS_64(info->push_idx, I40IWQPC_PPIDX) |
2865               LS_64(info->push_mode_en, I40IWQPC_PMENA);
2866
2867         set_64bit_val(qp_ctx, 8, qp->sq_pa);
2868         set_64bit_val(qp_ctx, 16, qp->rq_pa);
2869
2870         qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2871               LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) |
2872               LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE);
2873
2874         set_64bit_val(qp_ctx,
2875                       128,
2876                       LS_64(info->err_rq_idx, I40IWQPC_ERR_RQ_IDX));
2877
2878         set_64bit_val(qp_ctx,
2879                       136,
2880                       LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) |
2881                       LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM));
2882
2883         set_64bit_val(qp_ctx,
2884                       168,
2885                       LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX));
2886         set_64bit_val(qp_ctx,
2887                       176,
2888                       LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) |
2889                       LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) |
2890                       LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) |
2891                       LS_64(vsi->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));
2892
2893         if (info->iwarp_info_valid) {
2894                 qw0 |= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) |
2895                        LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
2896
2897                 qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX);
2898                 set_64bit_val(qp_ctx,
2899                               144,
2900                               LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) |
2901                               LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX));
2902                 set_64bit_val(qp_ctx,
2903                               152,
2904                               LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
2905
2906                 set_64bit_val(qp_ctx,
2907                               160,
2908                               LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
2909                               LS_64(iw->ird_size, I40IWQPC_IRDSIZE) |
2910                               LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) |
2911                               LS_64(iw->rd_enable, I40IWQPC_RDOK) |
2912                               LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) |
2913                               LS_64(iw->bind_en, I40IWQPC_BINDEN) |
2914                               LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) |
2915                               LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) |
2916                               LS_64((((vsi->stats_fcn_id_alloc) &&
2917                                       (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
2918                                     I40IWQPC_USESTATSINSTANCE) |
2919                               LS_64(1, I40IWQPC_IWARPMODE) |
2920                               LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) |
2921                               LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) |
2922                               LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) |
2923                               LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) |
2924                               LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET));
2925         }
2926         if (info->tcp_info_valid) {
2927                 qw0 |= LS_64(tcp->ipv4, I40IWQPC_IPV4) |
2928                        LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) |
2929                        LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) |
2930                        LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) |
2931                        LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) |
2932                        LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) |
2933                        LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH);
2934
2935                 qw3 |= LS_64(tcp->ttl, I40IWQPC_TTL) |
2936                        LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2937                        LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) |
2938                        LS_64(tcp->tos, I40IWQPC_TOS) |
2939                        LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) |
2940                        LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM);
2941
2942                 qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
2943                 set_64bit_val(qp_ctx,
2944                               32,
2945                               LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) |
2946                               LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3));
2947
2948                 set_64bit_val(qp_ctx,
2949                               40,
2950                               LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) |
2951                               LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1));
2952
2953                 set_64bit_val(qp_ctx,
2954                               48,
2955                               LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) |
2956                                 LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) |
2957                                 LS_64(tcp->arp_idx, I40IWQPC_ARPIDX));
2958
2959                 qw7 |= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) |
2960                        LS_64(tcp->wscale, I40IWQPC_WSCALE) |
2961                        LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) |
2962                        LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) |
2963                        LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) |
2964                        LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) |
2965                        LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE);
2966
2967                 set_64bit_val(qp_ctx,
2968                               72,
2969                               LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) |
2970                               LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE));
2971                 set_64bit_val(qp_ctx,
2972                               80,
2973                               LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) |
2974                               LS_64(tcp->snd_wnd, I40IWQPC_SNDWND));
2975
2976                 set_64bit_val(qp_ctx,
2977                               88,
2978                               LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) |
2979                               LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND));
2980                 set_64bit_val(qp_ctx,
2981                               96,
2982                               LS_64(tcp->snd_max, I40IWQPC_SNDMAX) |
2983                               LS_64(tcp->snd_una, I40IWQPC_SNDUNA));
2984                 set_64bit_val(qp_ctx,
2985                               104,
2986                               LS_64(tcp->srtt, I40IWQPC_SRTT) |
2987                               LS_64(tcp->rtt_var, I40IWQPC_RTTVAR));
2988                 set_64bit_val(qp_ctx,
2989                               112,
2990                               LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) |
2991                               LS_64(tcp->cwnd, I40IWQPC_CWND));
2992                 set_64bit_val(qp_ctx,
2993                               120,
2994                               LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) |
2995                               LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2));
2996                 set_64bit_val(qp_ctx,
2997                               128,
2998                               LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) |
2999                               LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH));
3000                 set_64bit_val(qp_ctx,
3001                               184,
3002                               LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) |
3003                               LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));
3004                 set_64bit_val(qp_ctx,
3005                               192,
3006                               LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) |
3007                               LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));
3008         }
3009
3010         set_64bit_val(qp_ctx, 0, qw0);
3011         set_64bit_val(qp_ctx, 24, qw3);
3012         set_64bit_val(qp_ctx, 56, qw7);
3013
3014         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE",
3015                         qp_ctx, I40IW_QP_CTX_SIZE);
3016         return 0;
3017 }
3018
3019 /**
3020  * i40iw_sc_alloc_stag - mr stag alloc
3021  * @dev: sc device struct
3022  * @info: stag info
3023  * @scratch: u64 saved to be used during cqp completion
3024  * @post_sq: flag for cqp db to ring
3025  */
3026 static enum i40iw_status_code i40iw_sc_alloc_stag(
3027                                 struct i40iw_sc_dev *dev,
3028                                 struct i40iw_allocate_stag_info *info,
3029                                 u64 scratch,
3030                                 bool post_sq)
3031 {
3032         u64 *wqe;
3033         struct i40iw_sc_cqp *cqp;
3034         u64 header;
3035         enum i40iw_page_size page_size;
3036
3037         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3038         cqp = dev->cqp;
3039         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3040         if (!wqe)
3041                 return I40IW_ERR_RING_FULL;
3042         set_64bit_val(wqe,
3043                       8,
3044                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) |
3045                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN));
3046         set_64bit_val(wqe,
3047                       16,
3048                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3049         set_64bit_val(wqe,
3050                       40,
3051                       LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));
3052
3053         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3054                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3055                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3056                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
3057                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
3058                  LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3059                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
3060                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
3061                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3062
3063         i40iw_insert_wqe_hdr(wqe, header);
3064
3065         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",
3066                         wqe, I40IW_CQP_WQE_SIZE * 8);
3067
3068         if (post_sq)
3069                 i40iw_sc_cqp_post_sq(cqp);
3070         return 0;
3071 }
3072
3073 /**
3074  * i40iw_sc_mr_reg_non_shared - non-shared mr registration
3075  * @dev: sc device struct
3076  * @info: mr info
3077  * @scratch: u64 saved to be used during cqp completion
3078  * @post_sq: flag for cqp db to ring
3079  */
3080 static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
3081                                 struct i40iw_sc_dev *dev,
3082                                 struct i40iw_reg_ns_stag_info *info,
3083                                 u64 scratch,
3084                                 bool post_sq)
3085 {
3086         u64 *wqe;
3087         u64 temp;
3088         struct i40iw_sc_cqp *cqp;
3089         u64 header;
3090         u32 pble_obj_cnt;
3091         bool remote_access;
3092         u8 addr_type;
3093         enum i40iw_page_size page_size;
3094
3095         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3096         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
3097                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
3098                 remote_access = true;
3099         else
3100                 remote_access = false;
3101
3102         pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
3103
3104         if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt))
3105                 return I40IW_ERR_INVALID_PBLE_INDEX;
3106
3107         cqp = dev->cqp;
3108         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3109         if (!wqe)
3110                 return I40IW_ERR_RING_FULL;
3111
3112         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3113         set_64bit_val(wqe, 0, temp);
3114
3115         set_64bit_val(wqe,
3116                       8,
3117                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) |
3118                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3119
3120         set_64bit_val(wqe,
3121                       16,
3122                       LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) |
3123                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3124         if (!info->chunk_size) {
3125                 set_64bit_val(wqe, 32, info->reg_addr_pa);
3126                 set_64bit_val(wqe, 48, 0);
3127         } else {
3128                 set_64bit_val(wqe, 32, 0);
3129                 set_64bit_val(wqe, 48, info->first_pm_pbl_index);
3130         }
3131         set_64bit_val(wqe, 40, info->hmc_fcn_index);
3132         set_64bit_val(wqe, 56, 0);
3133
3134         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
3135         header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |
3136                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3137                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
3138                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
3139                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3140                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3141                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3142                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
3143                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
3144                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3145
3146         i40iw_insert_wqe_hdr(wqe, header);
3147
3148         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",
3149                         wqe, I40IW_CQP_WQE_SIZE * 8);
3150
3151         if (post_sq)
3152                 i40iw_sc_cqp_post_sq(cqp);
3153         return 0;
3154 }
3155
3156 /**
3157  * i40iw_sc_mr_reg_shared - registered shared memory region
3158  * @dev: sc device struct
3159  * @info: info for shared memory registeration
3160  * @scratch: u64 saved to be used during cqp completion
3161  * @post_sq: flag for cqp db to ring
3162  */
3163 static enum i40iw_status_code i40iw_sc_mr_reg_shared(
3164                                         struct i40iw_sc_dev *dev,
3165                                         struct i40iw_register_shared_stag *info,
3166                                         u64 scratch,
3167                                         bool post_sq)
3168 {
3169         u64 *wqe;
3170         struct i40iw_sc_cqp *cqp;
3171         u64 temp, va64, fbo, header;
3172         u32 va32;
3173         bool remote_access;
3174         u8 addr_type;
3175
3176         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
3177                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
3178                 remote_access = true;
3179         else
3180                 remote_access = false;
3181         cqp = dev->cqp;
3182         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3183         if (!wqe)
3184                 return I40IW_ERR_RING_FULL;
3185         va64 = (uintptr_t)(info->va);
3186         va32 = (u32)(va64 & 0x00000000FFFFFFFF);
3187         fbo = (u64)(va32 & (4096 - 1));
3188
3189         set_64bit_val(wqe,
3190                       0,
3191                       (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo));
3192
3193         set_64bit_val(wqe,
3194                       8,
3195                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3196         temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) |
3197                LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) |
3198                LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);
3199         set_64bit_val(wqe, 16, temp);
3200
3201         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
3202         header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) |
3203                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3204                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3205                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3206                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3207                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3208
3209         i40iw_insert_wqe_hdr(wqe, header);
3210
3211         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",
3212                         wqe, I40IW_CQP_WQE_SIZE * 8);
3213
3214         if (post_sq)
3215                 i40iw_sc_cqp_post_sq(cqp);
3216         return 0;
3217 }
3218
3219 /**
3220  * i40iw_sc_dealloc_stag - deallocate stag
3221  * @dev: sc device struct
3222  * @info: dealloc stag info
3223  * @scratch: u64 saved to be used during cqp completion
3224  * @post_sq: flag for cqp db to ring
3225  */
3226 static enum i40iw_status_code i40iw_sc_dealloc_stag(
3227                                         struct i40iw_sc_dev *dev,
3228                                         struct i40iw_dealloc_stag_info *info,
3229                                         u64 scratch,
3230                                         bool post_sq)
3231 {
3232         u64 header;
3233         u64 *wqe;
3234         struct i40iw_sc_cqp *cqp;
3235
3236         cqp = dev->cqp;
3237         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3238         if (!wqe)
3239                 return I40IW_ERR_RING_FULL;
3240         set_64bit_val(wqe,
3241                       8,
3242                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3243         set_64bit_val(wqe,
3244                       16,
3245                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3246
3247         header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3248                  LS_64(info->mr, I40IW_CQPSQ_STAG_MR) |
3249                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3250
3251         i40iw_insert_wqe_hdr(wqe, header);
3252
3253         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",
3254                         wqe, I40IW_CQP_WQE_SIZE * 8);
3255
3256         if (post_sq)
3257                 i40iw_sc_cqp_post_sq(cqp);
3258         return 0;
3259 }
3260
3261 /**
3262  * i40iw_sc_query_stag - query hardware for stag
3263  * @dev: sc device struct
3264  * @scratch: u64 saved to be used during cqp completion
3265  * @stag_index: stag index for query
3266  * @post_sq: flag for cqp db to ring
3267  */
3268 static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev,
3269                                                   u64 scratch,
3270                                                   u32 stag_index,
3271                                                   bool post_sq)
3272 {
3273         u64 header;
3274         u64 *wqe;
3275         struct i40iw_sc_cqp *cqp;
3276
3277         cqp = dev->cqp;
3278         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3279         if (!wqe)
3280                 return I40IW_ERR_RING_FULL;
3281         set_64bit_val(wqe,
3282                       16,
3283                       LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));
3284
3285         header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) |
3286                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3287
3288         i40iw_insert_wqe_hdr(wqe, header);
3289
3290         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",
3291                         wqe, I40IW_CQP_WQE_SIZE * 8);
3292
3293         if (post_sq)
3294                 i40iw_sc_cqp_post_sq(cqp);
3295         return 0;
3296 }
3297
3298 /**
3299  * i40iw_sc_mw_alloc - mw allocate
3300  * @dev: sc device struct
3301  * @scratch: u64 saved to be used during cqp completion
3302  * @mw_stag_index:stag index
3303  * @pd_id: pd is for this mw
3304  * @post_sq: flag for cqp db to ring
3305  */
3306 static enum i40iw_status_code i40iw_sc_mw_alloc(
3307                                         struct i40iw_sc_dev *dev,
3308                                         u64 scratch,
3309                                         u32 mw_stag_index,
3310                                         u16 pd_id,
3311                                         bool post_sq)
3312 {
3313         u64 header;
3314         struct i40iw_sc_cqp *cqp;
3315         u64 *wqe;
3316
3317         cqp = dev->cqp;
3318         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3319         if (!wqe)
3320                 return I40IW_ERR_RING_FULL;
3321         set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));
3322         set_64bit_val(wqe,
3323                       16,
3324                       LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));
3325
3326         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3327                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3328
3329         i40iw_insert_wqe_hdr(wqe, header);
3330
3331         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",
3332                         wqe, I40IW_CQP_WQE_SIZE * 8);
3333
3334         if (post_sq)
3335                 i40iw_sc_cqp_post_sq(cqp);
3336         return 0;
3337 }
3338
3339 /**
3340  * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
3341  * @qp: sc qp struct
3342  * @info: fast mr info
3343  * @post_sq: flag for cqp db to ring
3344  */
3345 enum i40iw_status_code i40iw_sc_mr_fast_register(
3346                                 struct i40iw_sc_qp *qp,
3347                                 struct i40iw_fast_reg_stag_info *info,
3348                                 bool post_sq)
3349 {
3350         u64 temp, header;
3351         u64 *wqe;
3352         u32 wqe_idx;
3353         enum i40iw_page_size page_size;
3354
3355         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3356         wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
3357                                          0, info->wr_id);
3358         if (!wqe)
3359                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3360
3361         i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
3362                     __func__, info->wr_id, wqe_idx,
3363                     &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
3364         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3365         set_64bit_val(wqe, 0, temp);
3366
3367         temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI);
3368         set_64bit_val(wqe,
3369                       8,
3370                       LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) |
3371                       LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));
3372
3373         set_64bit_val(wqe,
3374                       16,
3375                       info->total_len |
3376                       LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));
3377
3378         header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) |
3379                  LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) |
3380                  LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |
3381                  LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) |
3382                  LS_64(page_size, I40IWQPSQ_HPAGESIZE) |
3383                  LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) |
3384                  LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) |
3385                  LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
3386                  LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
3387                  LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
3388                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3389
3390         i40iw_insert_wqe_hdr(wqe, header);
3391
3392         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE",
3393                         wqe, I40IW_QP_WQE_MIN_SIZE);
3394
3395         if (post_sq)
3396                 i40iw_qp_post_wr(&qp->qp_uk);
3397         return 0;
3398 }
3399
3400 /**
3401  * i40iw_sc_send_lsmm - send last streaming mode message
3402  * @qp: sc qp struct
3403  * @lsmm_buf: buffer with lsmm message
3404  * @size: size of lsmm buffer
3405  * @stag: stag of lsmm buffer
3406  */
3407 static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,
3408                                void *lsmm_buf,
3409                                u32 size,
3410                                i40iw_stag stag)
3411 {
3412         u64 *wqe;
3413         u64 header;
3414         struct i40iw_qp_uk *qp_uk;
3415
3416         qp_uk = &qp->qp_uk;
3417         wqe = qp_uk->sq_base->elem;
3418
3419         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3420
3421         set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG)));
3422
3423         set_64bit_val(wqe, 16, 0);
3424
3425         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3426                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3427                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3428                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3429
3430         i40iw_insert_wqe_hdr(wqe, header);
3431
3432         i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",
3433                         wqe, I40IW_QP_WQE_MIN_SIZE);
3434 }
3435
3436 /**
3437  * i40iw_sc_send_lsmm_nostag - for privilege qp
3438  * @qp: sc qp struct
3439  * @lsmm_buf: buffer with lsmm message
3440  * @size: size of lsmm buffer
3441  */
3442 static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,
3443                                       void *lsmm_buf,
3444                                       u32 size)
3445 {
3446         u64 *wqe;
3447         u64 header;
3448         struct i40iw_qp_uk *qp_uk;
3449
3450         qp_uk = &qp->qp_uk;
3451         wqe = qp_uk->sq_base->elem;
3452
3453         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3454
3455         set_64bit_val(wqe, 8, size);
3456
3457         set_64bit_val(wqe, 16, 0);
3458
3459         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3460                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3461                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3462                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3463
3464         i40iw_insert_wqe_hdr(wqe, header);
3465
3466         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
3467                         wqe, I40IW_QP_WQE_MIN_SIZE);
3468 }
3469
3470 /**
3471  * i40iw_sc_send_rtt - send last read0 or write0
3472  * @qp: sc qp struct
3473  * @read: Do read0 or write0
3474  */
3475 static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)
3476 {
3477         u64 *wqe;
3478         u64 header;
3479         struct i40iw_qp_uk *qp_uk;
3480
3481         qp_uk = &qp->qp_uk;
3482         wqe = qp_uk->sq_base->elem;
3483
3484         set_64bit_val(wqe, 0, 0);
3485         set_64bit_val(wqe, 8, 0);
3486         set_64bit_val(wqe, 16, 0);
3487         if (read) {
3488                 header = LS_64(0x1234, I40IWQPSQ_REMSTAG) |
3489                          LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) |
3490                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3491                 set_64bit_val(wqe, 8, ((u64)0xabcd << 32));
3492         } else {
3493                 header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
3494                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3495         }
3496
3497         i40iw_insert_wqe_hdr(wqe, header);
3498
3499         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE",
3500                         wqe, I40IW_QP_WQE_MIN_SIZE);
3501 }
3502
3503 /**
3504  * i40iw_sc_post_wqe0 - send wqe with opcode
3505  * @qp: sc qp struct
3506  * @opcode: opcode to use for wqe0
3507  */
3508 static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode)
3509 {
3510         u64 *wqe;
3511         u64 header;
3512         struct i40iw_qp_uk *qp_uk;
3513
3514         qp_uk = &qp->qp_uk;
3515         wqe = qp_uk->sq_base->elem;
3516
3517         if (!wqe)
3518                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3519         switch (opcode) {
3520         case I40IWQP_OP_NOP:
3521                 set_64bit_val(wqe, 0, 0);
3522                 set_64bit_val(wqe, 8, 0);
3523                 set_64bit_val(wqe, 16, 0);
3524                 header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
3525                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3526
3527                 i40iw_insert_wqe_hdr(wqe, header);
3528                 break;
3529         case I40IWQP_OP_RDMA_SEND:
3530                 set_64bit_val(wqe, 0, 0);
3531                 set_64bit_val(wqe, 8, 0);
3532                 set_64bit_val(wqe, 16, 0);
3533                 header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3534                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) |
3535                          LS_64(1, I40IWQPSQ_STREAMMODE) |
3536                          LS_64(1, I40IWQPSQ_WAITFORRCVPDU);
3537
3538                 i40iw_insert_wqe_hdr(wqe, header);
3539                 break;
3540         default:
3541                 i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n",
3542                             __func__);
3543                 break;
3544         }
3545         return 0;
3546 }
3547
3548 /**
3549  * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
3550  * @dev : ptr to i40iw_dev struct
3551  * @hmc_fn_id: hmc function id
3552  */
3553 enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
3554 {
3555         struct i40iw_hmc_info *hmc_info;
3556         struct i40iw_dma_mem query_fpm_mem;
3557         struct i40iw_virt_mem virt_mem;
3558         struct i40iw_vfdev *vf_dev = NULL;
3559         u32 mem_size;
3560         enum i40iw_status_code ret_code = 0;
3561         bool poll_registers = true;
3562         u16 iw_vf_idx;
3563         u8 wait_type;
3564
3565         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3566             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3567                 return I40IW_ERR_INVALID_HMCFN_ID;
3568
3569         i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id,
3570                     dev->hmc_fn_id);
3571         if (hmc_fn_id == dev->hmc_fn_id) {
3572                 hmc_info = dev->hmc_info;
3573                 query_fpm_mem.pa = dev->fpm_query_buf_pa;
3574                 query_fpm_mem.va = dev->fpm_query_buf;
3575         } else {
3576                 vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id);
3577                 if (!vf_dev)
3578                         return I40IW_ERR_INVALID_VF_ID;
3579
3580                 hmc_info = &vf_dev->hmc_info;
3581                 iw_vf_idx = vf_dev->iw_vf_idx;
3582                 i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev,
3583                             hmc_info, hmc_info->hmc_obj);
3584                 if (!vf_dev->fpm_query_buf) {
3585                         if (!dev->vf_fpm_query_buf[iw_vf_idx].va) {
3586                                 ret_code = i40iw_alloc_query_fpm_buf(dev,
3587                                                                      &dev->vf_fpm_query_buf[iw_vf_idx]);
3588                                 if (ret_code)
3589                                         return ret_code;
3590                         }
3591                         vf_dev->fpm_query_buf = dev->vf_fpm_query_buf[iw_vf_idx].va;
3592                         vf_dev->fpm_query_buf_pa = dev->vf_fpm_query_buf[iw_vf_idx].pa;
3593                 }
3594                 query_fpm_mem.pa = vf_dev->fpm_query_buf_pa;
3595                 query_fpm_mem.va = vf_dev->fpm_query_buf;
3596                 /**
3597                  * It is HARDWARE specific:
3598                  * this call is done by PF for VF and
3599                  * i40iw_sc_query_fpm_values needs ccq poll
3600                  * because PF ccq is already created.
3601                  */
3602                 poll_registers = false;
3603         }
3604
3605         hmc_info->hmc_fn_id = hmc_fn_id;
3606
3607         if (hmc_fn_id != dev->hmc_fn_id) {
3608                 ret_code =
3609                         i40iw_cqp_query_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3610         } else {
3611                 wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3612                             (u8)I40IW_CQP_WAIT_POLL_CQ;
3613
3614                 ret_code = i40iw_sc_query_fpm_values(
3615                                         dev->cqp,
3616                                         0,
3617                                         hmc_info->hmc_fn_id,
3618                                         &query_fpm_mem,
3619                                         true,
3620                                         wait_type);
3621         }
3622         if (ret_code)
3623                 return ret_code;
3624
3625         /* parse the fpm_query_buf and fill hmc obj info */
3626         ret_code =
3627                 i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,
3628                                              hmc_info,
3629                                              &dev->hmc_fpm_misc);
3630         if (ret_code)
3631                 return ret_code;
3632         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",
3633                         query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);
3634
3635         if (hmc_fn_id != dev->hmc_fn_id) {
3636                 i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3637
3638                 /* parse the fpm_commit_buf and fill hmc obj info */
3639                 i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt);
3640                 mem_size = sizeof(struct i40iw_hmc_sd_entry) *
3641                            (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index);
3642                 ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
3643                 if (ret_code)
3644                         return ret_code;
3645                 hmc_info->sd_table.sd_entry = virt_mem.va;
3646         }
3647
3648         return ret_code;
3649 }
3650
3651 /**
3652  * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
3653  * populates fpm base address in hmc_info
3654  * @dev : ptr to i40iw_dev struct
3655  * @hmc_fn_id: hmc function id
3656  */
3657 static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,
3658                                                         u8 hmc_fn_id)
3659 {
3660         struct i40iw_hmc_info *hmc_info;
3661         struct i40iw_hmc_obj_info *obj_info;
3662         u64 *buf;
3663         struct i40iw_dma_mem commit_fpm_mem;
3664         u32 i, j;
3665         enum i40iw_status_code ret_code = 0;
3666         bool poll_registers = true;
3667         u8 wait_type;
3668
3669         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3670             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3671                 return I40IW_ERR_INVALID_HMCFN_ID;
3672
3673         if (hmc_fn_id == dev->hmc_fn_id) {
3674                 hmc_info = dev->hmc_info;
3675         } else {
3676                 hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id);
3677                 poll_registers = false;
3678         }
3679         if (!hmc_info)
3680                 return I40IW_ERR_BAD_PTR;
3681
3682         obj_info = hmc_info->hmc_obj;
3683         buf = dev->fpm_commit_buf;
3684
3685         /* copy cnt values in commit buf */
3686         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE;
3687              i++, j += 8)
3688                 set_64bit_val(buf, j, (u64)obj_info[i].cnt);
3689
3690         set_64bit_val(buf, 40, 0);   /* APBVT rsvd */
3691
3692         commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
3693         commit_fpm_mem.va = dev->fpm_commit_buf;
3694         wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3695                         (u8)I40IW_CQP_WAIT_POLL_CQ;
3696         ret_code = i40iw_sc_commit_fpm_values(
3697                                         dev->cqp,
3698                                         0,
3699                                         hmc_info->hmc_fn_id,
3700                                         &commit_fpm_mem,
3701                                         true,
3702                                         wait_type);
3703
3704         /* parse the fpm_commit_buf and fill hmc obj info */
3705         if (!ret_code)
3706                 ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf,
3707                                                          hmc_info->hmc_obj,
3708                                                          &hmc_info->sd_table.sd_cnt);
3709
3710         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",
3711                         commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);
3712
3713         return ret_code;
3714 }
3715
3716 /**
3717  * cqp_sds_wqe_fill - fill cqp wqe doe sd
3718  * @cqp: struct for cqp hw
3719  * @info; sd info for wqe
3720  * @scratch: u64 saved to be used during cqp completion
3721  */
3722 static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,
3723                                                struct i40iw_update_sds_info *info,
3724                                                u64 scratch)
3725 {
3726         u64 data;
3727         u64 header;
3728         u64 *wqe;
3729         int mem_entries, wqe_entries;
3730         struct i40iw_dma_mem *sdbuf = &cqp->sdbuf;
3731         u64 offset;
3732         u32 wqe_idx;
3733
3734         wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
3735         if (!wqe)
3736                 return I40IW_ERR_RING_FULL;
3737
3738         I40IW_CQP_INIT_WQE(wqe);
3739         wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
3740         mem_entries = info->cnt - wqe_entries;
3741
3742         header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) |
3743                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
3744                  LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);
3745
3746         if (mem_entries) {
3747                 offset = wqe_idx * I40IW_UPDATE_SD_BUF_SIZE;
3748                 memcpy((char *)sdbuf->va + offset, &info->entry[3],
3749                        mem_entries << 4);
3750                 data = (u64)sdbuf->pa + offset;
3751         } else {
3752                 data = 0;
3753         }
3754         data |= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);
3755
3756         set_64bit_val(wqe, 16, data);
3757
3758         switch (wqe_entries) {
3759         case 3:
3760                 set_64bit_val(wqe, 48,
3761                               (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3762                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3763
3764                 set_64bit_val(wqe, 56, info->entry[2].data);
3765                 /* fallthrough */
3766         case 2:
3767                 set_64bit_val(wqe, 32,
3768                               (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3769                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3770
3771                 set_64bit_val(wqe, 40, info->entry[1].data);
3772                 /* fallthrough */
3773         case 1:
3774                 set_64bit_val(wqe, 0,
3775                               LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));
3776
3777                 set_64bit_val(wqe, 8, info->entry[0].data);
3778                 break;
3779         default:
3780                 break;
3781         }
3782
3783         i40iw_insert_wqe_hdr(wqe, header);
3784
3785         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",
3786                         wqe, I40IW_CQP_WQE_SIZE * 8);
3787         return 0;
3788 }
3789
3790 /**
3791  * i40iw_update_pe_sds - cqp wqe for sd
3792  * @dev: ptr to i40iw_dev struct
3793  * @info: sd info for sd's
3794  * @scratch: u64 saved to be used during cqp completion
3795  */
3796 static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev,
3797                                                   struct i40iw_update_sds_info *info,
3798                                                   u64 scratch)
3799 {
3800         struct i40iw_sc_cqp *cqp = dev->cqp;
3801         enum i40iw_status_code ret_code;
3802
3803         ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
3804         if (!ret_code)
3805                 i40iw_sc_cqp_post_sq(cqp);
3806
3807         return ret_code;
3808 }
3809
3810 /**
3811  * i40iw_update_sds_noccq - update sd before ccq created
3812  * @dev: sc device struct
3813  * @info: sd info for sd's
3814  */
3815 enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
3816                                               struct i40iw_update_sds_info *info)
3817 {
3818         u32 error, val, tail;
3819         struct i40iw_sc_cqp *cqp = dev->cqp;
3820         enum i40iw_status_code ret_code;
3821
3822         ret_code = cqp_sds_wqe_fill(cqp, info, 0);
3823         if (ret_code)
3824                 return ret_code;
3825         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3826         if (error)
3827                 return I40IW_ERR_CQP_COMPL_ERROR;
3828
3829         i40iw_sc_cqp_post_sq(cqp);
3830         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
3831
3832         return ret_code;
3833 }
3834
3835 /**
3836  * i40iw_sc_suspend_qp - suspend qp for param change
3837  * @cqp: struct for cqp hw
3838  * @qp: sc qp struct
3839  * @scratch: u64 saved to be used during cqp completion
3840  */
3841 enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
3842                                            struct i40iw_sc_qp *qp,
3843                                            u64 scratch)
3844 {
3845         u64 header;
3846         u64 *wqe;
3847
3848         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3849         if (!wqe)
3850                 return I40IW_ERR_RING_FULL;
3851         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) |
3852                  LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) |
3853                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3854
3855         i40iw_insert_wqe_hdr(wqe, header);
3856
3857         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",
3858                         wqe, I40IW_CQP_WQE_SIZE * 8);
3859
3860         i40iw_sc_cqp_post_sq(cqp);
3861         return 0;
3862 }
3863
3864 /**
3865  * i40iw_sc_resume_qp - resume qp after suspend
3866  * @cqp: struct for cqp hw
3867  * @qp: sc qp struct
3868  * @scratch: u64 saved to be used during cqp completion
3869  */
3870 enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
3871                                           struct i40iw_sc_qp *qp,
3872                                           u64 scratch)
3873 {
3874         u64 header;
3875         u64 *wqe;
3876
3877         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3878         if (!wqe)
3879                 return I40IW_ERR_RING_FULL;
3880         set_64bit_val(wqe,
3881                       16,
3882                         LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));
3883
3884         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) |
3885                  LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) |
3886                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3887
3888         i40iw_insert_wqe_hdr(wqe, header);
3889
3890         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",
3891                         wqe, I40IW_CQP_WQE_SIZE * 8);
3892
3893         i40iw_sc_cqp_post_sq(cqp);
3894         return 0;
3895 }
3896
3897 /**
3898  * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
3899  * @cqp: struct for cqp hw
3900  * @scratch: u64 saved to be used during cqp completion
3901  * @hmc_fn_id: hmc function id
3902  * @post_sq: flag for cqp db to ring
3903  * @poll_registers: flag to poll register for cqp completion
3904  */
3905 enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(
3906                                         struct i40iw_sc_cqp *cqp,
3907                                         u64 scratch,
3908                                         u8 hmc_fn_id,
3909                                         bool post_sq,
3910                                         bool poll_registers)
3911 {
3912         u64 header;
3913         u64 *wqe;
3914         u32 tail, val, error;
3915         enum i40iw_status_code ret_code = 0;
3916
3917         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3918         if (!wqe)
3919                 return I40IW_ERR_RING_FULL;
3920         set_64bit_val(wqe,
3921                       16,
3922                       LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
3923
3924         header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) |
3925                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3926
3927         i40iw_insert_wqe_hdr(wqe, header);
3928
3929         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
3930                         wqe, I40IW_CQP_WQE_SIZE * 8);
3931         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3932         if (error) {
3933                 ret_code = I40IW_ERR_CQP_COMPL_ERROR;
3934                 return ret_code;
3935         }
3936         if (post_sq) {
3937                 i40iw_sc_cqp_post_sq(cqp);
3938                 if (poll_registers)
3939                         /* check for cqp sq tail update */
3940                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
3941                 else
3942                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
3943                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
3944                                                                  NULL);
3945         }
3946
3947         return ret_code;
3948 }
3949
3950 /**
3951  * i40iw_ring_full - check if cqp ring is full
3952  * @cqp: struct for cqp hw
3953  */
3954 static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp)
3955 {
3956         return I40IW_RING_FULL_ERR(cqp->sq_ring);
3957 }
3958
3959 /**
3960  * i40iw_est_sd - returns approximate number of SDs for HMC
3961  * @dev: sc device struct
3962  * @hmc_info: hmc structure, size and count for HMC objects
3963  */
3964 static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info)
3965 {
3966         int i;
3967         u64 size = 0;
3968         u64 sd;
3969
3970         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++)
3971                 size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size;
3972
3973         if (dev->is_pf)
3974                 size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3975
3976         if (size & 0x1FFFFF)
3977                 sd = (size >> 21) + 1; /* add 1 for remainder */
3978         else
3979                 sd = size >> 21;
3980
3981         if (!dev->is_pf) {
3982                 /* 2MB alignment for VF PBLE HMC */
3983                 size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3984                 if (size & 0x1FFFFF)
3985                         sd += (size >> 21) + 1; /* add 1 for remainder */
3986                 else
3987                         sd += size >> 21;
3988         }
3989
3990         return sd;
3991 }
3992
3993 /**
3994  * i40iw_config_fpm_values - configure HMC objects
3995  * @dev: sc device struct
3996  * @qp_count: desired qp count
3997  */
3998 enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)
3999 {
4000         struct i40iw_virt_mem virt_mem;
4001         u32 i, mem_size;
4002         u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
4003         u64 sd_needed;
4004         u32 loop_count = 0;
4005
4006         struct i40iw_hmc_info *hmc_info;
4007         struct i40iw_hmc_fpm_misc *hmc_fpm_misc;
4008         enum i40iw_status_code ret_code = 0;
4009
4010         hmc_info = dev->hmc_info;
4011         hmc_fpm_misc = &dev->hmc_fpm_misc;
4012
4013         ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id);
4014         if (ret_code) {
4015                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4016                             "i40iw_sc_init_iw_hmc returned error_code = %d\n",
4017                             ret_code);
4018                 return ret_code;
4019         }
4020
4021         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
4022                 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
4023         sd_needed = i40iw_est_sd(dev, hmc_info);
4024         i40iw_debug(dev, I40IW_DEBUG_HMC,
4025                     "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
4026                     __func__, sd_needed, hmc_info->first_sd_index);
4027         i40iw_debug(dev, I40IW_DEBUG_HMC,
4028                     "%s: sd count %d where max sd is %d\n",
4029                     __func__, hmc_info->sd_table.sd_cnt,
4030                     hmc_fpm_misc->max_sds);
4031
4032         qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt);
4033         qpwantedoriginal = qpwanted;
4034         mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt;
4035         pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;
4036
4037         i40iw_debug(dev, I40IW_DEBUG_HMC,
4038                     "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
4039                     qp_count, hmc_fpm_misc->max_sds,
4040                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt,
4041                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt,
4042                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt,
4043                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);
4044
4045         do {
4046                 ++loop_count;
4047                 hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;
4048                 hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt =
4049                         min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt);
4050                 hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */
4051                 hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt =
4052                                         qpwanted * hmc_fpm_misc->ht_multiplier;
4053                 hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt =
4054                         hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt;
4055                 hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
4056                 hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
4057
4058                 hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt =
4059                         roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted);
4060                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt =
4061                         roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted);
4062                 hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
4063                         hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
4064                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
4065                         hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
4066                 hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt =
4067                         ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket;
4068                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;
4069                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;
4070                 hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;
4071
4072                 /* How much memory is needed for all the objects. */
4073                 sd_needed = i40iw_est_sd(dev, hmc_info);
4074                 if ((loop_count > 1000) ||
4075                     ((!(loop_count % 10)) &&
4076                     (qpwanted > qpwantedoriginal * 2 / 3))) {
4077                         if (qpwanted > FPM_MULTIPLIER)
4078                                 qpwanted = roundup_pow_of_two(qpwanted -
4079                                                               FPM_MULTIPLIER);
4080                         qpwanted >>= 1;
4081                 }
4082                 if (mrwanted > FPM_MULTIPLIER * 10)
4083                         mrwanted -= FPM_MULTIPLIER * 10;
4084                 if (pblewanted > FPM_MULTIPLIER * 1000)
4085                         pblewanted -= FPM_MULTIPLIER * 1000;
4086         } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
4087
4088         i40iw_debug(dev, I40IW_DEBUG_HMC,
4089                     "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
4090                     loop_count, sd_needed,
4091                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt,
4092                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt,
4093                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt,
4094                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt);
4095
4096         ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id);
4097         if (ret_code) {
4098                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4099                             "configure_iw_fpm returned error_code[x%08X]\n",
4100                             i40iw_rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));
4101                 return ret_code;
4102         }
4103
4104         mem_size = sizeof(struct i40iw_hmc_sd_entry) *
4105                    (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
4106         ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
4107         if (ret_code) {
4108                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4109                             "%s: failed to allocate memory for sd_entry buffer\n",
4110                             __func__);
4111                 return ret_code;
4112         }
4113         hmc_info->sd_table.sd_entry = virt_mem.va;
4114
4115         return ret_code;
4116 }
4117
4118 /**
4119  * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
4120  * @dev: rdma device
4121  * @pcmdinfo: cqp command info
4122  */
4123 static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
4124                                                  struct cqp_commands_info *pcmdinfo)
4125 {
4126         enum i40iw_status_code status;
4127         struct i40iw_dma_mem values_mem;
4128
4129         dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
4130         switch (pcmdinfo->cqp_cmd) {
4131         case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:
4132                 status = i40iw_sc_del_local_mac_ipaddr_entry(
4133                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp,
4134                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch,
4135                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx,
4136                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count,
4137                                 pcmdinfo->post_sq);
4138                 break;
4139         case OP_CEQ_DESTROY:
4140                 status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
4141                                               pcmdinfo->in.u.ceq_destroy.scratch,
4142                                               pcmdinfo->post_sq);
4143                 break;
4144         case OP_AEQ_DESTROY:
4145                 status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
4146                                               pcmdinfo->in.u.aeq_destroy.scratch,
4147                                               pcmdinfo->post_sq);
4148
4149                 break;
4150         case OP_DELETE_ARP_CACHE_ENTRY:
4151                 status = i40iw_sc_del_arp_cache_entry(
4152                                 pcmdinfo->in.u.del_arp_cache_entry.cqp,
4153                                 pcmdinfo->in.u.del_arp_cache_entry.scratch,
4154                                 pcmdinfo->in.u.del_arp_cache_entry.arp_index,
4155                                 pcmdinfo->post_sq);
4156                 break;
4157         case OP_MANAGE_APBVT_ENTRY:
4158                 status = i40iw_sc_manage_apbvt_entry(
4159                                 pcmdinfo->in.u.manage_apbvt_entry.cqp,
4160                                 &pcmdinfo->in.u.manage_apbvt_entry.info,
4161                                 pcmdinfo->in.u.manage_apbvt_entry.scratch,
4162                                 pcmdinfo->post_sq);
4163                 break;
4164         case OP_CEQ_CREATE:
4165                 status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
4166                                              pcmdinfo->in.u.ceq_create.scratch,
4167                                              pcmdinfo->post_sq);
4168                 break;
4169         case OP_AEQ_CREATE:
4170                 status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
4171                                              pcmdinfo->in.u.aeq_create.scratch,
4172                                              pcmdinfo->post_sq);
4173                 break;
4174         case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:
4175                 status = i40iw_sc_alloc_local_mac_ipaddr_entry(
4176                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp,
4177                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch,
4178                                 pcmdinfo->post_sq);
4179                 break;
4180         case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:
4181                 status = i40iw_sc_add_local_mac_ipaddr_entry(
4182                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp,
4183                                 &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info,
4184                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch,
4185                                 pcmdinfo->post_sq);
4186                 break;
4187         case OP_MANAGE_QHASH_TABLE_ENTRY:
4188                 status = i40iw_sc_manage_qhash_table_entry(
4189                                 pcmdinfo->in.u.manage_qhash_table_entry.cqp,
4190                                 &pcmdinfo->in.u.manage_qhash_table_entry.info,
4191                                 pcmdinfo->in.u.manage_qhash_table_entry.scratch,
4192                                 pcmdinfo->post_sq);
4193
4194                 break;
4195         case OP_QP_MODIFY:
4196                 status = i40iw_sc_qp_modify(
4197                                 pcmdinfo->in.u.qp_modify.qp,
4198                                 &pcmdinfo->in.u.qp_modify.info,
4199                                 pcmdinfo->in.u.qp_modify.scratch,
4200                                 pcmdinfo->post_sq);
4201
4202                 break;
4203         case OP_QP_UPLOAD_CONTEXT:
4204                 status = i40iw_sc_qp_upload_context(
4205                                 pcmdinfo->in.u.qp_upload_context.dev,
4206                                 &pcmdinfo->in.u.qp_upload_context.info,
4207                                 pcmdinfo->in.u.qp_upload_context.scratch,
4208                                 pcmdinfo->post_sq);
4209
4210                 break;
4211         case OP_CQ_CREATE:
4212                 status = i40iw_sc_cq_create(
4213                                 pcmdinfo->in.u.cq_create.cq,
4214                                 pcmdinfo->in.u.cq_create.scratch,
4215                                 pcmdinfo->in.u.cq_create.check_overflow,
4216                                 pcmdinfo->post_sq);
4217                 break;
4218         case OP_CQ_DESTROY:
4219                 status = i40iw_sc_cq_destroy(
4220                                 pcmdinfo->in.u.cq_destroy.cq,
4221                                 pcmdinfo->in.u.cq_destroy.scratch,
4222                                 pcmdinfo->post_sq);
4223
4224                 break;
4225         case OP_QP_CREATE:
4226                 status = i40iw_sc_qp_create(
4227                                 pcmdinfo->in.u.qp_create.qp,
4228                                 &pcmdinfo->in.u.qp_create.info,
4229                                 pcmdinfo->in.u.qp_create.scratch,
4230                                 pcmdinfo->post_sq);
4231                 break;
4232         case OP_QP_DESTROY:
4233                 status = i40iw_sc_qp_destroy(
4234                                 pcmdinfo->in.u.qp_destroy.qp,
4235                                 pcmdinfo->in.u.qp_destroy.scratch,
4236                                 pcmdinfo->in.u.qp_destroy.remove_hash_idx,
4237                                 pcmdinfo->in.u.qp_destroy.
4238                                 ignore_mw_bnd,
4239                                 pcmdinfo->post_sq);
4240
4241                 break;
4242         case OP_ALLOC_STAG:
4243                 status = i40iw_sc_alloc_stag(
4244                                 pcmdinfo->in.u.alloc_stag.dev,
4245                                 &pcmdinfo->in.u.alloc_stag.info,
4246                                 pcmdinfo->in.u.alloc_stag.scratch,
4247                                 pcmdinfo->post_sq);
4248                 break;
4249         case OP_MR_REG_NON_SHARED:
4250                 status = i40iw_sc_mr_reg_non_shared(
4251                                 pcmdinfo->in.u.mr_reg_non_shared.dev,
4252                                 &pcmdinfo->in.u.mr_reg_non_shared.info,
4253                                 pcmdinfo->in.u.mr_reg_non_shared.scratch,
4254                                 pcmdinfo->post_sq);
4255
4256                 break;
4257         case OP_DEALLOC_STAG:
4258                 status = i40iw_sc_dealloc_stag(
4259                                 pcmdinfo->in.u.dealloc_stag.dev,
4260                                 &pcmdinfo->in.u.dealloc_stag.info,
4261                                 pcmdinfo->in.u.dealloc_stag.scratch,
4262                                 pcmdinfo->post_sq);
4263
4264                 break;
4265         case OP_MW_ALLOC:
4266                 status = i40iw_sc_mw_alloc(
4267                                 pcmdinfo->in.u.mw_alloc.dev,
4268                                 pcmdinfo->in.u.mw_alloc.scratch,
4269                                 pcmdinfo->in.u.mw_alloc.mw_stag_index,
4270                                 pcmdinfo->in.u.mw_alloc.pd_id,
4271                                 pcmdinfo->post_sq);
4272
4273                 break;
4274         case OP_QP_FLUSH_WQES:
4275                 status = i40iw_sc_qp_flush_wqes(
4276                                 pcmdinfo->in.u.qp_flush_wqes.qp,
4277                                 &pcmdinfo->in.u.qp_flush_wqes.info,
4278                                 pcmdinfo->in.u.qp_flush_wqes.
4279                                 scratch, pcmdinfo->post_sq);
4280                 break;
4281         case OP_GEN_AE:
4282                 status = i40iw_sc_gen_ae(
4283                                 pcmdinfo->in.u.gen_ae.qp,
4284                                 &pcmdinfo->in.u.gen_ae.info,
4285                                 pcmdinfo->in.u.gen_ae.scratch,
4286                                 pcmdinfo->post_sq);
4287                 break;
4288         case OP_ADD_ARP_CACHE_ENTRY:
4289                 status = i40iw_sc_add_arp_cache_entry(
4290                                 pcmdinfo->in.u.add_arp_cache_entry.cqp,
4291                                 &pcmdinfo->in.u.add_arp_cache_entry.info,
4292                                 pcmdinfo->in.u.add_arp_cache_entry.scratch,
4293                                 pcmdinfo->post_sq);
4294                 break;
4295         case OP_MANAGE_PUSH_PAGE:
4296                 status = i40iw_sc_manage_push_page(
4297                                 pcmdinfo->in.u.manage_push_page.cqp,
4298                                 &pcmdinfo->in.u.manage_push_page.info,
4299                                 pcmdinfo->in.u.manage_push_page.scratch,
4300                                 pcmdinfo->post_sq);
4301                 break;
4302         case OP_UPDATE_PE_SDS:
4303                 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
4304                 status = i40iw_update_pe_sds(
4305                                 pcmdinfo->in.u.update_pe_sds.dev,
4306                                 &pcmdinfo->in.u.update_pe_sds.info,
4307                                 pcmdinfo->in.u.update_pe_sds.
4308                                 scratch);
4309
4310                 break;
4311         case OP_MANAGE_HMC_PM_FUNC_TABLE:
4312                 status = i40iw_sc_manage_hmc_pm_func_table(
4313                                 pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
4314                                 pcmdinfo->in.u.manage_hmc_pm.scratch,
4315                                 (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
4316                                 pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
4317                                 true);
4318                 break;
4319         case OP_SUSPEND:
4320                 status = i40iw_sc_suspend_qp(
4321                                 pcmdinfo->in.u.suspend_resume.cqp,
4322                                 pcmdinfo->in.u.suspend_resume.qp,
4323                                 pcmdinfo->in.u.suspend_resume.scratch);
4324                 break;
4325         case OP_RESUME:
4326                 status = i40iw_sc_resume_qp(
4327                                 pcmdinfo->in.u.suspend_resume.cqp,
4328                                 pcmdinfo->in.u.suspend_resume.qp,
4329                                 pcmdinfo->in.u.suspend_resume.scratch);
4330                 break;
4331         case OP_MANAGE_VF_PBLE_BP:
4332                 status = i40iw_manage_vf_pble_bp(
4333                                 pcmdinfo->in.u.manage_vf_pble_bp.cqp,
4334                                 &pcmdinfo->in.u.manage_vf_pble_bp.info,
4335                                 pcmdinfo->in.u.manage_vf_pble_bp.scratch, true);
4336                 break;
4337         case OP_QUERY_FPM_VALUES:
4338                 values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa;
4339                 values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va;
4340                 status = i40iw_sc_query_fpm_values(
4341                                 pcmdinfo->in.u.query_fpm_values.cqp,
4342                                 pcmdinfo->in.u.query_fpm_values.scratch,
4343                                 pcmdinfo->in.u.query_fpm_values.hmc_fn_id,
4344                                 &values_mem, true, I40IW_CQP_WAIT_EVENT);
4345                 break;
4346         case OP_COMMIT_FPM_VALUES:
4347                 values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa;
4348                 values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va;
4349                 status = i40iw_sc_commit_fpm_values(
4350                                 pcmdinfo->in.u.commit_fpm_values.cqp,
4351                                 pcmdinfo->in.u.commit_fpm_values.scratch,
4352                                 pcmdinfo->in.u.commit_fpm_values.hmc_fn_id,
4353                                 &values_mem,
4354                                 true,
4355                                 I40IW_CQP_WAIT_EVENT);
4356                 break;
4357         case OP_QUERY_RDMA_FEATURES:
4358                 values_mem.pa = pcmdinfo->in.u.query_rdma_features.cap_pa;
4359                 values_mem.va = pcmdinfo->in.u.query_rdma_features.cap_va;
4360                 status = i40iw_sc_query_rdma_features(
4361                         pcmdinfo->in.u.query_rdma_features.cqp, &values_mem,
4362                         pcmdinfo->in.u.query_rdma_features.scratch);
4363                 break;
4364         default:
4365                 status = I40IW_NOT_SUPPORTED;
4366                 break;
4367         }
4368
4369         return status;
4370 }
4371
4372 /**
4373  * i40iw_process_cqp_cmd - process all cqp commands
4374  * @dev: sc device struct
4375  * @pcmdinfo: cqp command info
4376  */
4377 enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
4378                                              struct cqp_commands_info *pcmdinfo)
4379 {
4380         enum i40iw_status_code status = 0;
4381         unsigned long flags;
4382
4383         spin_lock_irqsave(&dev->cqp_lock, flags);
4384         if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
4385                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4386         else
4387                 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
4388         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4389         return status;
4390 }
4391
4392 /**
4393  * i40iw_process_bh - called from tasklet for cqp list
4394  * @dev: sc device struct
4395  */
4396 enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)
4397 {
4398         enum i40iw_status_code status = 0;
4399         struct cqp_commands_info *pcmdinfo;
4400         unsigned long flags;
4401
4402         spin_lock_irqsave(&dev->cqp_lock, flags);
4403         while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
4404                 pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
4405
4406                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4407                 if (status)
4408                         break;
4409         }
4410         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4411         return status;
4412 }
4413
4414 /**
4415  * i40iw_iwarp_opcode - determine if incoming is rdma layer
4416  * @info: aeq info for the packet
4417  * @pkt: packet for error
4418  */
4419 static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt)
4420 {
4421         __be16 *mpa;
4422         u32 opcode = 0xffffffff;
4423
4424         if (info->q2_data_written) {
4425                 mpa = (__be16 *)pkt;
4426                 opcode = ntohs(mpa[1]) & 0xf;
4427         }
4428         return opcode;
4429 }
4430
4431 /**
4432  * i40iw_locate_mpa - return pointer to mpa in the pkt
4433  * @pkt: packet with data
4434  */
4435 static u8 *i40iw_locate_mpa(u8 *pkt)
4436 {
4437         /* skip over ethernet header */
4438         pkt += I40IW_MAC_HLEN;
4439
4440         /* Skip over IP and TCP headers */
4441         pkt += 4 * (pkt[0] & 0x0f);
4442         pkt += 4 * ((pkt[12] >> 4) & 0x0f);
4443         return pkt;
4444 }
4445
4446 /**
4447  * i40iw_setup_termhdr - termhdr for terminate pkt
4448  * @qp: sc qp ptr for pkt
4449  * @hdr: term hdr
4450  * @opcode: flush opcode for termhdr
4451  * @layer_etype: error layer + error type
4452  * @err: error cod ein the header
4453  */
4454 static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp,
4455                                 struct i40iw_terminate_hdr *hdr,
4456                                 enum i40iw_flush_opcode opcode,
4457                                 u8 layer_etype,
4458                                 u8 err)
4459 {
4460         qp->flush_code = opcode;
4461         hdr->layer_etype = layer_etype;
4462         hdr->error_code = err;
4463 }
4464
4465 /**
4466  * i40iw_bld_terminate_hdr - build terminate message header
4467  * @qp: qp associated with received terminate AE
4468  * @info: the struct contiaing AE information
4469  */
4470 static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
4471                                    struct i40iw_aeqe_info *info)
4472 {
4473         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4474         u16 ddp_seg_len;
4475         int copy_len = 0;
4476         u8 is_tagged = 0;
4477         u32 opcode;
4478         struct i40iw_terminate_hdr *termhdr;
4479
4480         termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf;
4481         memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
4482
4483         if (info->q2_data_written) {
4484                 /* Use data from offending packet to fill in ddp & rdma hdrs */
4485                 pkt = i40iw_locate_mpa(pkt);
4486                 ddp_seg_len = ntohs(*(__be16 *)pkt);
4487                 if (ddp_seg_len) {
4488                         copy_len = 2;
4489                         termhdr->hdrct = DDP_LEN_FLAG;
4490                         if (pkt[2] & 0x80) {
4491                                 is_tagged = 1;
4492                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
4493                                         copy_len += TERM_DDP_LEN_TAGGED;
4494                                         termhdr->hdrct |= DDP_HDR_FLAG;
4495                                 }
4496                         } else {
4497                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
4498                                         copy_len += TERM_DDP_LEN_UNTAGGED;
4499                                         termhdr->hdrct |= DDP_HDR_FLAG;
4500                                 }
4501
4502                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
4503                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
4504                                                 copy_len += TERM_RDMA_LEN;
4505                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
4506                                         }
4507                                 }
4508                         }
4509                 }
4510         }
4511
4512         opcode = i40iw_iwarp_opcode(info, pkt);
4513
4514         switch (info->ae_id) {
4515         case I40IW_AE_AMP_UNALLOCATED_STAG:
4516                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4517                 if (opcode == I40IW_OP_TYPE_RDMA_WRITE)
4518                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4519                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);
4520                 else
4521                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4522                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4523                 break;
4524         case I40IW_AE_AMP_BOUNDS_VIOLATION:
4525                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4526                 if (info->q2_data_written)
4527                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4528                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);
4529                 else
4530                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4531                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);
4532                 break;
4533         case I40IW_AE_AMP_BAD_PD:
4534                 switch (opcode) {
4535                 case I40IW_OP_TYPE_RDMA_WRITE:
4536                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4537                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);
4538                         break;
4539                 case I40IW_OP_TYPE_SEND_INV:
4540                 case I40IW_OP_TYPE_SEND_SOL_INV:
4541                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4542                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);
4543                         break;
4544                 default:
4545                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4546                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);
4547                 }
4548                 break;
4549         case I40IW_AE_AMP_INVALID_STAG:
4550                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4551                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4552                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4553                 break;
4554         case I40IW_AE_AMP_BAD_QP:
4555                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4556                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4557                 break;
4558         case I40IW_AE_AMP_BAD_STAG_KEY:
4559         case I40IW_AE_AMP_BAD_STAG_INDEX:
4560                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4561                 switch (opcode) {
4562                 case I40IW_OP_TYPE_SEND_INV:
4563                 case I40IW_OP_TYPE_SEND_SOL_INV:
4564                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4565                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);
4566                         break;
4567                 default:
4568                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4569                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG);
4570                 }
4571                 break;
4572         case I40IW_AE_AMP_RIGHTS_VIOLATION:
4573         case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
4574         case I40IW_AE_PRIV_OPERATION_DENIED:
4575                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4576                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4577                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS);
4578                 break;
4579         case I40IW_AE_AMP_TO_WRAP:
4580                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4581                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4582                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);
4583                 break;
4584         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4585                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4586                                     (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
4587                 break;
4588         case I40IW_AE_LLP_SEGMENT_TOO_LARGE:
4589         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
4590                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4591                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4592                 break;
4593         case I40IW_AE_LCE_QP_CATASTROPHIC:
4594         case I40IW_AE_DDP_NO_L_BIT:
4595                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4596                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4597                 break;
4598         case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:
4599                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4600                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);
4601                 break;
4602         case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
4603                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4604                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4605                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);
4606                 break;
4607         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
4608                 if (is_tagged)
4609                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4610                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);
4611                 else
4612                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4613                                             (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);
4614                 break;
4615         case I40IW_AE_DDP_UBE_INVALID_MO:
4616                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4617                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);
4618                 break;
4619         case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
4620                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4621                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);
4622                 break;
4623         case I40IW_AE_DDP_UBE_INVALID_QN:
4624                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4625                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4626                 break;
4627         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4628                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4629                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);
4630                 break;
4631         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4632                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4633                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);
4634                 break;
4635         default:
4636                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4637                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);
4638                 break;
4639         }
4640
4641         if (copy_len)
4642                 memcpy(termhdr + 1, pkt, copy_len);
4643
4644         return sizeof(struct i40iw_terminate_hdr) + copy_len;
4645 }
4646
4647 /**
4648  * i40iw_terminate_send_fin() - Send fin for terminate message
4649  * @qp: qp associated with received terminate AE
4650  */
4651 void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)
4652 {
4653         /* Send the fin only */
4654         i40iw_term_modify_qp(qp,
4655                              I40IW_QP_STATE_TERMINATE,
4656                              I40IWQP_TERM_SEND_FIN_ONLY,
4657                              0);
4658 }
4659
4660 /**
4661  * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
4662  * @qp: qp associated with received terminate AE
4663  * @info: the struct contiaing AE information
4664  */
4665 void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4666 {
4667         u8 termlen = 0;
4668
4669         if (qp->term_flags & I40IW_TERM_SENT)
4670                 return;         /* Sanity check */
4671
4672         /* Eventtype can change from bld_terminate_hdr */
4673         qp->eventtype = TERM_EVENT_QP_FATAL;
4674         termlen = i40iw_bld_terminate_hdr(qp, info);
4675         i40iw_terminate_start_timer(qp);
4676         qp->term_flags |= I40IW_TERM_SENT;
4677         i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,
4678                              I40IWQP_TERM_SEND_TERM_ONLY, termlen);
4679 }
4680
4681 /**
4682  * i40iw_terminate_received - handle terminate received AE
4683  * @qp: qp associated with received terminate AE
4684  * @info: the struct contiaing AE information
4685  */
4686 void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4687 {
4688         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4689         __be32 *mpa;
4690         u8 ddp_ctl;
4691         u8 rdma_ctl;
4692         u16 aeq_id = 0;
4693         struct i40iw_terminate_hdr *termhdr;
4694
4695         mpa = (__be32 *)i40iw_locate_mpa(pkt);
4696         if (info->q2_data_written) {
4697                 /* did not validate the frame - do it now */
4698                 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
4699                 rdma_ctl = ntohl(mpa[0]) & 0xff;
4700                 if ((ddp_ctl & 0xc0) != 0x40)
4701                         aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;
4702                 else if ((ddp_ctl & 0x03) != 1)
4703                         aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;
4704                 else if (ntohl(mpa[2]) != 2)
4705                         aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;
4706                 else if (ntohl(mpa[3]) != 1)
4707                         aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;
4708                 else if (ntohl(mpa[4]) != 0)
4709                         aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;
4710                 else if ((rdma_ctl & 0xc0) != 0x40)
4711                         aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
4712
4713                 info->ae_id = aeq_id;
4714                 if (info->ae_id) {
4715                         /* Bad terminate recvd - send back a terminate */
4716                         i40iw_terminate_connection(qp, info);
4717                         return;
4718                 }
4719         }
4720
4721         qp->term_flags |= I40IW_TERM_RCVD;
4722         qp->eventtype = TERM_EVENT_QP_FATAL;
4723         termhdr = (struct i40iw_terminate_hdr *)&mpa[5];
4724         if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
4725             termhdr->layer_etype == RDMAP_REMOTE_OP) {
4726                 i40iw_terminate_done(qp, 0);
4727         } else {
4728                 i40iw_terminate_start_timer(qp);
4729                 i40iw_terminate_send_fin(qp);
4730         }
4731 }
4732
4733 /**
4734  * i40iw_sc_vsi_init - Initialize virtual device
4735  * @vsi: pointer to the vsi structure
4736  * @info: parameters to initialize vsi
4737  **/
4738 void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)
4739 {
4740         int i;
4741
4742         vsi->dev = info->dev;
4743         vsi->back_vsi = info->back_vsi;
4744         vsi->mtu = info->params->mtu;
4745         vsi->exception_lan_queue = info->exception_lan_queue;
4746         i40iw_fill_qos_list(info->params->qs_handle_list);
4747
4748         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
4749                 vsi->qos[i].qs_handle = info->params->qs_handle_list[i];
4750                 i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i,
4751                             vsi->qos[i].qs_handle);
4752                 spin_lock_init(&vsi->qos[i].lock);
4753                 INIT_LIST_HEAD(&vsi->qos[i].qplist);
4754         }
4755 }
4756
4757 /**
4758  * i40iw_hw_stats_init - Initiliaze HW stats table
4759  * @stats: pestat struct
4760  * @fcn_idx: PCI fn id
4761  * @is_pf: Is it a PF?
4762  *
4763  * Populate the HW stats table with register offset addr for each
4764  * stats. And start the perioidic stats timer.
4765  */
4766 void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
4767 {
4768         u32 stats_reg_offset;
4769         u32 stats_index;
4770         struct i40iw_dev_hw_stats_offsets *stats_table =
4771                 &stats->hw_stats_offsets;
4772         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4773
4774         if (is_pf) {
4775                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4776                                 I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
4777                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4778                                 I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
4779                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4780                                 I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
4781                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4782                                 I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
4783                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4784                                 I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
4785                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4786                                 I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
4787                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4788                                 I40E_GLPES_PFTCPRTXSEG(fcn_idx);
4789                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4790                                 I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
4791                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4792                                 I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
4793
4794                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4795                                 I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
4796                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4797                                 I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
4798                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4799                                 I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
4800                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4801                                 I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
4802                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4803                                 I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
4804                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4805                                 I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
4806                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4807                                 I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
4808                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4809                                 I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
4810                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4811                                 I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
4812                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4813                                 I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
4814                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4815                                 I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
4816                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4817                                 I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
4818                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4819                                 I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
4820                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4821                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4822                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4823                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4824                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4825                                 I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
4826                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4827                                 I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
4828                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4829                                 I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
4830                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4831                                 I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
4832                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4833                                 I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
4834                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4835                                 I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
4836                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4837                                 I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
4838                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4839                                 I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
4840                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4841                                 I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
4842                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4843                                 I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
4844                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4845                                 I40E_GLPES_PFRDMAVINVLO(fcn_idx);
4846         } else {
4847                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4848                                 I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
4849                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4850                                 I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
4851                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4852                                 I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
4853                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4854                                 I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
4855                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4856                                 I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
4857                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4858                                 I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
4859                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4860                                 I40E_GLPES_VFTCPRTXSEG(fcn_idx);
4861                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4862                                 I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
4863                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4864                                 I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
4865
4866                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4867                                 I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
4868                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4869                                 I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
4870                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4871                                 I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
4872                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4873                                 I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
4874                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4875                                 I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
4876                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4877                                 I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
4878                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4879                                 I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
4880                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4881                                 I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
4882                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4883                                 I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
4884                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4885                                 I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
4886                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4887                                 I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
4888                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4889                                 I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
4890                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4891                                 I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
4892                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4893                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4894                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4895                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4896                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4897                                 I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
4898                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4899                                 I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
4900                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4901                                 I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
4902                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4903                                 I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
4904                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4905                                 I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
4906                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4907                                 I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
4908                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4909                                 I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
4910                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4911                                 I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
4912                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4913                                 I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
4914                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4915                                 I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
4916                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4917                                 I40E_GLPES_VFRDMAVINVLO(fcn_idx);
4918         }
4919
4920         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4921              stats_index++) {
4922                 stats_reg_offset = stats_table->stats_offset_64[stats_index];
4923                 last_rd_stats->stats_value_64[stats_index] =
4924                         readq(stats->hw->hw_addr + stats_reg_offset);
4925         }
4926
4927         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4928              stats_index++) {
4929                 stats_reg_offset = stats_table->stats_offset_32[stats_index];
4930                 last_rd_stats->stats_value_32[stats_index] =
4931                         i40iw_rd32(stats->hw, stats_reg_offset);
4932         }
4933 }
4934
4935 /**
4936  * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
4937  * @stat: pestat struct
4938  * @index: index in HW stats table which contains offset reg-addr
4939  * @value: hw stats value
4940  */
4941 void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
4942                             enum i40iw_hw_stats_index_32b index,
4943                             u64 *value)
4944 {
4945         struct i40iw_dev_hw_stats_offsets *stats_table =
4946                 &stats->hw_stats_offsets;
4947         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4948         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4949         u64 new_stats_value = 0;
4950         u32 stats_reg_offset = stats_table->stats_offset_32[index];
4951
4952         new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset);
4953         /*roll-over case */
4954         if (new_stats_value < last_rd_stats->stats_value_32[index])
4955                 hw_stats->stats_value_32[index] += new_stats_value;
4956         else
4957                 hw_stats->stats_value_32[index] +=
4958                         new_stats_value - last_rd_stats->stats_value_32[index];
4959         last_rd_stats->stats_value_32[index] = new_stats_value;
4960         *value = hw_stats->stats_value_32[index];
4961 }
4962
4963 /**
4964  * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
4965  * @stats: pestat struct
4966  * @index: index in HW stats table which contains offset reg-addr
4967  * @value: hw stats value
4968  */
4969 void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
4970                             enum i40iw_hw_stats_index_64b index,
4971                             u64 *value)
4972 {
4973         struct i40iw_dev_hw_stats_offsets *stats_table =
4974                 &stats->hw_stats_offsets;
4975         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4976         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4977         u64 new_stats_value = 0;
4978         u32 stats_reg_offset = stats_table->stats_offset_64[index];
4979
4980         new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset);
4981         /*roll-over case */
4982         if (new_stats_value < last_rd_stats->stats_value_64[index])
4983                 hw_stats->stats_value_64[index] += new_stats_value;
4984         else
4985                 hw_stats->stats_value_64[index] +=
4986                         new_stats_value - last_rd_stats->stats_value_64[index];
4987         last_rd_stats->stats_value_64[index] = new_stats_value;
4988         *value = hw_stats->stats_value_64[index];
4989 }
4990
4991 /**
4992  * i40iw_hw_stats_read_all - read all HW stat counters
4993  * @stats: pestat struct
4994  * @stats_values: hw stats structure
4995  *
4996  * Read all the HW stat counters and populates hw_stats structure
4997  * of passed-in vsi's pestat as well as copy created in stat_values.
4998  */
4999 void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
5000                              struct i40iw_dev_hw_stats *stats_values)
5001 {
5002         u32 stats_index;
5003         unsigned long flags;
5004
5005         spin_lock_irqsave(&stats->lock, flags);
5006
5007         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
5008              stats_index++)
5009                 i40iw_hw_stats_read_32(stats, stats_index,
5010                                        &stats_values->stats_value_32[stats_index]);
5011         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
5012              stats_index++)
5013                 i40iw_hw_stats_read_64(stats, stats_index,
5014                                        &stats_values->stats_value_64[stats_index]);
5015         spin_unlock_irqrestore(&stats->lock, flags);
5016 }
5017
5018 /**
5019  * i40iw_hw_stats_refresh_all - Update all HW stats structs
5020  * @stats: pestat struct
5021  *
5022  * Read all the HW stats counters to refresh values in hw_stats structure
5023  * of passed-in dev's pestat
5024  */
5025 void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
5026 {
5027         u64 stats_value;
5028         u32 stats_index;
5029         unsigned long flags;
5030
5031         spin_lock_irqsave(&stats->lock, flags);
5032
5033         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
5034              stats_index++)
5035                 i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
5036         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
5037              stats_index++)
5038                 i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
5039         spin_unlock_irqrestore(&stats->lock, flags);
5040 }
5041
5042 /**
5043  * i40iw_get_fcn_id - Return the function id
5044  * @dev: pointer to the device
5045  */
5046 static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
5047 {
5048         u8 fcn_id = I40IW_INVALID_FCN_ID;
5049         u8 i;
5050
5051         for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
5052                 if (!dev->fcn_id_array[i]) {
5053                         fcn_id = i;
5054                         dev->fcn_id_array[i] = true;
5055                         break;
5056                 }
5057         return fcn_id;
5058 }
5059
5060 /**
5061  * i40iw_vsi_stats_init - Initialize the vsi statistics
5062  * @vsi: pointer to the vsi structure
5063  * @info: The info structure used for initialization
5064  */
5065 enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)
5066 {
5067         u8 fcn_id = info->fcn_id;
5068
5069         if (info->alloc_fcn_id)
5070                 fcn_id = i40iw_get_fcn_id(vsi->dev);
5071
5072         if (fcn_id == I40IW_INVALID_FCN_ID)
5073                 return I40IW_ERR_NOT_READY;
5074
5075         vsi->pestat = info->pestat;
5076         vsi->pestat->hw = vsi->dev->hw;
5077         vsi->pestat->vsi = vsi;
5078
5079         if (info->stats_initialize) {
5080                 i40iw_hw_stats_init(vsi->pestat, fcn_id, true);
5081                 spin_lock_init(&vsi->pestat->lock);
5082                 i40iw_hw_stats_start_timer(vsi);
5083         }
5084         vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
5085         vsi->fcn_id = fcn_id;
5086         return I40IW_SUCCESS;
5087 }
5088
5089 /**
5090  * i40iw_vsi_stats_free - Free the vsi stats
5091  * @vsi: pointer to the vsi structure
5092  */
5093 void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
5094 {
5095         u8 fcn_id = vsi->fcn_id;
5096
5097         if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT)
5098                 vsi->dev->fcn_id_array[fcn_id] = false;
5099         i40iw_hw_stats_stop_timer(vsi);
5100 }
5101
5102 static struct i40iw_cqp_ops iw_cqp_ops = {
5103         .cqp_init = i40iw_sc_cqp_init,
5104         .cqp_create = i40iw_sc_cqp_create,
5105         .cqp_post_sq = i40iw_sc_cqp_post_sq,
5106         .cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe,
5107         .cqp_destroy = i40iw_sc_cqp_destroy,
5108         .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done
5109 };
5110
5111 static struct i40iw_ccq_ops iw_ccq_ops = {
5112         .ccq_init = i40iw_sc_ccq_init,
5113         .ccq_create = i40iw_sc_ccq_create,
5114         .ccq_destroy = i40iw_sc_ccq_destroy,
5115         .ccq_create_done = i40iw_sc_ccq_create_done,
5116         .ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info,
5117         .ccq_arm = i40iw_sc_ccq_arm
5118 };
5119
5120 static struct i40iw_ceq_ops iw_ceq_ops = {
5121         .ceq_init = i40iw_sc_ceq_init,
5122         .ceq_create = i40iw_sc_ceq_create,
5123         .cceq_create_done = i40iw_sc_cceq_create_done,
5124         .cceq_destroy_done = i40iw_sc_cceq_destroy_done,
5125         .cceq_create = i40iw_sc_cceq_create,
5126         .ceq_destroy = i40iw_sc_ceq_destroy,
5127         .process_ceq = i40iw_sc_process_ceq
5128 };
5129
5130 static struct i40iw_aeq_ops iw_aeq_ops = {
5131         .aeq_init = i40iw_sc_aeq_init,
5132         .aeq_create = i40iw_sc_aeq_create,
5133         .aeq_destroy = i40iw_sc_aeq_destroy,
5134         .get_next_aeqe = i40iw_sc_get_next_aeqe,
5135         .repost_aeq_entries = i40iw_sc_repost_aeq_entries,
5136         .aeq_create_done = i40iw_sc_aeq_create_done,
5137         .aeq_destroy_done = i40iw_sc_aeq_destroy_done
5138 };
5139
5140 /* iwarp pd ops */
5141 static struct i40iw_pd_ops iw_pd_ops = {
5142         .pd_init = i40iw_sc_pd_init,
5143 };
5144
5145 static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
5146         .qp_init = i40iw_sc_qp_init,
5147         .qp_create = i40iw_sc_qp_create,
5148         .qp_modify = i40iw_sc_qp_modify,
5149         .qp_destroy = i40iw_sc_qp_destroy,
5150         .qp_flush_wqes = i40iw_sc_qp_flush_wqes,
5151         .qp_upload_context = i40iw_sc_qp_upload_context,
5152         .qp_setctx = i40iw_sc_qp_setctx,
5153         .qp_send_lsmm = i40iw_sc_send_lsmm,
5154         .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,
5155         .qp_send_rtt = i40iw_sc_send_rtt,
5156         .qp_post_wqe0 = i40iw_sc_post_wqe0,
5157         .iw_mr_fast_register = i40iw_sc_mr_fast_register
5158 };
5159
5160 static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
5161         .cq_init = i40iw_sc_cq_init,
5162         .cq_create = i40iw_sc_cq_create,
5163         .cq_destroy = i40iw_sc_cq_destroy,
5164         .cq_modify = i40iw_sc_cq_modify,
5165 };
5166
5167 static struct i40iw_mr_ops iw_mr_ops = {
5168         .alloc_stag = i40iw_sc_alloc_stag,
5169         .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,
5170         .mr_reg_shared = i40iw_sc_mr_reg_shared,
5171         .dealloc_stag = i40iw_sc_dealloc_stag,
5172         .query_stag = i40iw_sc_query_stag,
5173         .mw_alloc = i40iw_sc_mw_alloc
5174 };
5175
5176 static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
5177         .manage_push_page = i40iw_sc_manage_push_page,
5178         .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,
5179         .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,
5180         .commit_fpm_values = i40iw_sc_commit_fpm_values,
5181         .query_fpm_values = i40iw_sc_query_fpm_values,
5182         .static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated,
5183         .add_arp_cache_entry = i40iw_sc_add_arp_cache_entry,
5184         .del_arp_cache_entry = i40iw_sc_del_arp_cache_entry,
5185         .query_arp_cache_entry = i40iw_sc_query_arp_cache_entry,
5186         .manage_apbvt_entry = i40iw_sc_manage_apbvt_entry,
5187         .manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry,
5188         .alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry,
5189         .add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry,
5190         .del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry,
5191         .cqp_nop = i40iw_sc_cqp_nop,
5192         .commit_fpm_values_done = i40iw_sc_commit_fpm_values_done,
5193         .query_fpm_values_done = i40iw_sc_query_fpm_values_done,
5194         .manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done,
5195         .update_suspend_qp = i40iw_sc_suspend_qp,
5196         .update_resume_qp = i40iw_sc_resume_qp
5197 };
5198
5199 static struct i40iw_hmc_ops iw_hmc_ops = {
5200         .init_iw_hmc = i40iw_sc_init_iw_hmc,
5201         .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,
5202         .configure_iw_fpm = i40iw_sc_configure_iw_fpm,
5203         .parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf,
5204         .create_hmc_object = i40iw_sc_create_hmc_obj,
5205         .del_hmc_object = i40iw_sc_del_hmc_obj
5206 };
5207
5208 /**
5209  * i40iw_device_init - Initialize IWARP device
5210  * @dev: IWARP device pointer
5211  * @info: IWARP init info
5212  */
5213 enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
5214                                          struct i40iw_device_init_info *info)
5215 {
5216         u32 val;
5217         u32 vchnl_ver = 0;
5218         u16 hmc_fcn = 0;
5219         enum i40iw_status_code ret_code = 0;
5220         u8 db_size;
5221
5222         spin_lock_init(&dev->cqp_lock);
5223
5224         i40iw_device_init_uk(&dev->dev_uk);
5225
5226         dev->debug_mask = info->debug_mask;
5227
5228         dev->hmc_fn_id = info->hmc_fn_id;
5229         dev->is_pf = info->is_pf;
5230
5231         dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5232         dev->fpm_query_buf = info->fpm_query_buf;
5233
5234         dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5235         dev->fpm_commit_buf = info->fpm_commit_buf;
5236
5237         dev->hw = info->hw;
5238         dev->hw->hw_addr = info->bar0;
5239
5240         if (dev->is_pf) {
5241                 val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
5242                 dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
5243
5244                 val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL);
5245                 db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
5246                 if ((db_size != I40IW_PE_DB_SIZE_4M) &&
5247                     (db_size != I40IW_PE_DB_SIZE_8M)) {
5248                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5249                                     "%s: PE doorbell is not enabled in CSR val 0x%x\n",
5250                                     __func__, val);
5251                         ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;
5252                         return ret_code;
5253                 }
5254                 dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET;
5255                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf;
5256         } else {
5257                 dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET;
5258         }
5259
5260         dev->cqp_ops = &iw_cqp_ops;
5261         dev->ccq_ops = &iw_ccq_ops;
5262         dev->ceq_ops = &iw_ceq_ops;
5263         dev->aeq_ops = &iw_aeq_ops;
5264         dev->cqp_misc_ops = &iw_cqp_misc_ops;
5265         dev->iw_pd_ops = &iw_pd_ops;
5266         dev->iw_priv_qp_ops = &iw_priv_qp_ops;
5267         dev->iw_priv_cq_ops = &iw_priv_cq_ops;
5268         dev->mr_ops = &iw_mr_ops;
5269         dev->hmc_ops = &iw_hmc_ops;
5270         dev->vchnl_if.vchnl_send = info->vchnl_send;
5271         if (dev->vchnl_if.vchnl_send)
5272                 dev->vchnl_up = true;
5273         else
5274                 dev->vchnl_up = false;
5275         if (!dev->is_pf) {
5276                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf;
5277                 ret_code = i40iw_vchnl_vf_get_ver(dev, &vchnl_ver);
5278                 if (!ret_code) {
5279                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5280                                     "%s: Get Channel version rc = 0x%0x, version is %u\n",
5281                                 __func__, ret_code, vchnl_ver);
5282                         ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &hmc_fcn);
5283                         if (!ret_code) {
5284                                 i40iw_debug(dev, I40IW_DEBUG_DEV,
5285                                             "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
5286                                             __func__, ret_code, hmc_fcn);
5287                                 dev->hmc_fn_id = (u8)hmc_fcn;
5288                         }
5289                 }
5290         }
5291         dev->iw_vf_cqp_ops = &iw_vf_cqp_ops;
5292
5293         return ret_code;
5294 }