drivers/infiniband/hw/irdma/hmc.c

   1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
   2 /* Copyright (c) 2015 - 2021 Intel Corporation */
   3 #include "osdep.h"
   4 #include "status.h"
   5 #include "hmc.h"
   6 #include "defs.h"
   7 #include "type.h"
   8 #include "protos.h"
   9
  10 /**
  11  * irdma_find_sd_index_limit - finds segment descriptor index limit
  12  * @hmc_info: pointer to the HMC configuration information structure
  13  * @type: type of HMC resources we're searching
  14  * @idx: starting index for the object
  15  * @cnt: number of objects we're trying to create
  16  * @sd_idx: pointer to return index of the segment descriptor in question
  17  * @sd_limit: pointer to return the maximum number of segment descriptors
  18  *
  19  * This function calculates the segment descriptor index and index limit
  20  * for the resource defined by irdma_hmc_rsrc_type.
  21  */
  22
  23 static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
  24                                       u32 idx, u32 cnt, u32 *sd_idx,
  25                                       u32 *sd_limit)
  26 {
  27         u64 fpm_addr, fpm_limit;
  28
  29         fpm_addr = hmc_info->hmc_obj[(type)].base +
  30                    hmc_info->hmc_obj[type].size * idx;
  31         fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
  32         *sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);
  33         *sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);
  34         *sd_limit += 1;
  35 }
  36
  37 /**
  38  * irdma_find_pd_index_limit - finds page descriptor index limit
  39  * @hmc_info: pointer to the HMC configuration information struct
  40  * @type: HMC resource type we're examining
  41  * @idx: starting index for the object
  42  * @cnt: number of objects we're trying to create
  43  * @pd_idx: pointer to return page descriptor index
  44  * @pd_limit: pointer to return page descriptor index limit
  45  *
  46  * Calculates the page descriptor index and index limit for the resource
  47  * defined by irdma_hmc_rsrc_type.
  48  */
  49
  50 static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
  51                                       u32 idx, u32 cnt, u32 *pd_idx,
  52                                       u32 *pd_limit)
  53 {
  54         u64 fpm_adr, fpm_limit;
  55
  56         fpm_adr = hmc_info->hmc_obj[type].base +
  57                   hmc_info->hmc_obj[type].size * idx;
  58         fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
  59         *pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);
  60         *pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);
  61         *pd_limit += 1;
  62 }
  63
  64 /**
  65  * irdma_set_sd_entry - setup entry for sd programming
  66  * @pa: physical addr
  67  * @idx: sd index
  68  * @type: paged or direct sd
  69  * @entry: sd entry ptr
  70  */
  71 static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
  72                                struct irdma_update_sd_entry *entry)
  73 {
  74         entry->data = pa |
  75                       FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
  76                       FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
  77                                  type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |
  78                       FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);
  79
  80         entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
  81 }
  82
  83 /**
  84  * irdma_clr_sd_entry - setup entry for sd clear
  85  * @idx: sd index
  86  * @type: paged or direct sd
  87  * @entry: sd entry ptr
  88  */
  89 static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
  90                                struct irdma_update_sd_entry *entry)
  91 {
  92         entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
  93                       FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
  94                                  type == IRDMA_SD_TYPE_PAGED ? 0 : 1);
  95
  96         entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
  97 }
  98
  99 /**
 100  * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF
 101  * @dev: pointer to our device struct
 102  * @sd_idx: segment descriptor index
 103  * @pd_idx: page descriptor index
 104  */
 105 static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,
 106                                               u32 pd_idx)
 107 {
 108         u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |
 109                   FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |
 110                   FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);
 111
 112         writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]);
 113 }
 114
 115 /**
 116  * irdma_hmc_sd_one - setup 1 sd entry for cqp
 117  * @dev: pointer to the device structure
 118  * @hmc_fn_id: hmc's function id
 119  * @pa: physical addr
 120  * @sd_idx: sd index
 121  * @type: paged or direct sd
 122  * @setsd: flag to set or clear sd
 123  */
 124 enum irdma_status_code irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id,
 125                                         u64 pa, u32 sd_idx,
 126                                         enum irdma_sd_entry_type type,
 127                                         bool setsd)
 128 {
 129         struct irdma_update_sds_info sdinfo;
 130
 131         sdinfo.cnt = 1;
 132         sdinfo.hmc_fn_id = hmc_fn_id;
 133         if (setsd)
 134                 irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
 135         else
 136                 irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);
 137         return dev->cqp->process_cqp_sds(dev, &sdinfo);
 138 }
 139
 140 /**
 141  * irdma_hmc_sd_grp - setup group of sd entries for cqp
 142  * @dev: pointer to the device structure
 143  * @hmc_info: pointer to the HMC configuration information struct
 144  * @sd_index: sd index
 145  * @sd_cnt: number of sd entries
 146  * @setsd: flag to set or clear sd
 147  */
 148 static enum irdma_status_code irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
 149                                                struct irdma_hmc_info *hmc_info,
 150                                                u32 sd_index, u32 sd_cnt,
 151                                                bool setsd)
 152 {
 153         struct irdma_hmc_sd_entry *sd_entry;
 154         struct irdma_update_sds_info sdinfo = {};
 155         u64 pa;
 156         u32 i;
 157         enum irdma_status_code ret_code = 0;
 158
 159         sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
 160         for (i = sd_index; i < sd_index + sd_cnt; i++) {
 161                 sd_entry = &hmc_info->sd_table.sd_entry[i];
 162                 if (!sd_entry || (!sd_entry->valid && setsd) ||
 163                     (sd_entry->valid && !setsd))
 164                         continue;
 165                 if (setsd) {
 166                         pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
 167                                      sd_entry->u.pd_table.pd_page_addr.pa :
 168                                      sd_entry->u.bp.addr.pa;
 169                         irdma_set_sd_entry(pa, i, sd_entry->entry_type,
 170                                            &sdinfo.entry[sdinfo.cnt]);
 171                 } else {
 172                         irdma_clr_sd_entry(i, sd_entry->entry_type,
 173                                            &sdinfo.entry[sdinfo.cnt]);
 174                 }
 175                 sdinfo.cnt++;
 176                 if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {
 177                         ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
 178                         if (ret_code) {
 179                                 ibdev_dbg(to_ibdev(dev),
 180                                           "HMC: sd_programming failed err=%d\n",
 181                                           ret_code);
 182                                 return ret_code;
 183                         }
 184
 185                         sdinfo.cnt = 0;
 186                 }
 187         }
 188         if (sdinfo.cnt)
 189                 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
 190
 191         return ret_code;
 192 }
 193
 194 /**
 195  * irdma_hmc_finish_add_sd_reg - program sd entries for objects
 196  * @dev: pointer to the device structure
 197  * @info: create obj info
 198  */
 199 static enum irdma_status_code
 200 irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
 201                             struct irdma_hmc_create_obj_info *info)
 202 {
 203         if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
 204                 return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
 205
 206         if ((info->start_idx + info->count) >
 207             info->hmc_info->hmc_obj[info->rsrc_type].cnt)
 208                 return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
 209
 210         if (!info->add_sd_cnt)
 211                 return 0;
 212         return irdma_hmc_sd_grp(dev, info->hmc_info,
 213                                 info->hmc_info->sd_indexes[0], info->add_sd_cnt,
 214                                 true);
 215 }
 216
 217 /**
 218  * irdma_sc_create_hmc_obj - allocate backing store for hmc objects
 219  * @dev: pointer to the device structure
 220  * @info: pointer to irdma_hmc_create_obj_info struct
 221  *
 222  * This will allocate memory for PDs and backing pages and populate
 223  * the sd and pd entries.
 224  */
 225 enum irdma_status_code
 226 irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
 227                         struct irdma_hmc_create_obj_info *info)
 228 {
 229         struct irdma_hmc_sd_entry *sd_entry;
 230         u32 sd_idx, sd_lmt;
 231         u32 pd_idx = 0, pd_lmt = 0;
 232         u32 pd_idx1 = 0, pd_lmt1 = 0;
 233         u32 i, j;
 234         bool pd_error = false;
 235         enum irdma_status_code ret_code = 0;
 236
 237         if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
 238                 return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
 239
 240         if ((info->start_idx + info->count) >
 241             info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
 242                 ibdev_dbg(to_ibdev(dev),
 243                           "HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
 244                           info->rsrc_type, info->start_idx, info->count,
 245                           info->hmc_info->hmc_obj[info->rsrc_type].cnt);
 246                 return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
 247         }
 248
 249         irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
 250                                   info->start_idx, info->count, &sd_idx,
 251                                   &sd_lmt);
 252         if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
 253             sd_lmt > info->hmc_info->sd_table.sd_cnt) {
 254                 return IRDMA_ERR_INVALID_SD_INDEX;
 255         }
 256
 257         irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
 258                                   info->start_idx, info->count, &pd_idx,
 259                                   &pd_lmt);
 260
 261         for (j = sd_idx; j < sd_lmt; j++) {
 262                 ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j,
 263                                                     info->entry_type,
 264                                                     IRDMA_HMC_DIRECT_BP_SIZE);
 265                 if (ret_code)
 266                         goto exit_sd_error;
 267
 268                 sd_entry = &info->hmc_info->sd_table.sd_entry[j];
 269                 if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED &&
 270                     (dev->hmc_info == info->hmc_info &&
 271                      info->rsrc_type != IRDMA_HMC_IW_PBLE)) {
 272                         pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));
 273                         pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);
 274                         for (i = pd_idx1; i < pd_lmt1; i++) {
 275                                 /* update the pd table entry */
 276                                 ret_code = irdma_add_pd_table_entry(dev,
 277                                                                     info->hmc_info,
 278                                                                     i, NULL);
 279                                 if (ret_code) {
 280                                         pd_error = true;
 281                                         break;
 282                                 }
 283                         }
 284                         if (pd_error) {
 285                                 while (i && (i > pd_idx1)) {
 286                                         irdma_remove_pd_bp(dev, info->hmc_info,
 287                                                            i - 1);
 288                                         i--;
 289                                 }
 290                         }
 291                 }
 292                 if (sd_entry->valid)
 293                         continue;
 294
 295                 info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
 296                 info->add_sd_cnt++;
 297                 sd_entry->valid = true;
 298         }
 299         return irdma_hmc_finish_add_sd_reg(dev, info);
 300
 301 exit_sd_error:
 302         while (j && (j > sd_idx)) {
 303                 sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
 304                 switch (sd_entry->entry_type) {
 305                 case IRDMA_SD_TYPE_PAGED:
 306                         pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);
 307                         pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));
 308                         for (i = pd_idx1; i < pd_lmt1; i++)
 309                                 irdma_prep_remove_pd_page(info->hmc_info, i);
 310                         break;
 311                 case IRDMA_SD_TYPE_DIRECT:
 312                         irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
 313                         break;
 314                 default:
 315                         ret_code = IRDMA_ERR_INVALID_SD_TYPE;
 316                         break;
 317                 }
 318                 j--;
 319         }
 320
 321         return ret_code;
 322 }
 323
 324 /**
 325  * irdma_finish_del_sd_reg - delete sd entries for objects
 326  * @dev: pointer to the device structure
 327  * @info: dele obj info
 328  * @reset: true if called before reset
 329  */
 330 static enum irdma_status_code
 331 irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
 332                         struct irdma_hmc_del_obj_info *info, bool reset)
 333 {
 334         struct irdma_hmc_sd_entry *sd_entry;
 335         enum irdma_status_code ret_code = 0;
 336         u32 i, sd_idx;
 337         struct irdma_dma_mem *mem;
 338
 339         if (!reset)
 340                 ret_code = irdma_hmc_sd_grp(dev, info->hmc_info,
 341                                             info->hmc_info->sd_indexes[0],
 342                                             info->del_sd_cnt, false);
 343
 344         if (ret_code)
 345                 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n");
 346         for (i = 0; i < info->del_sd_cnt; i++) {
 347                 sd_idx = info->hmc_info->sd_indexes[i];
 348                 sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
 349                 mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
 350                               &sd_entry->u.pd_table.pd_page_addr :
 351                               &sd_entry->u.bp.addr;
 352
 353                 if (!mem || !mem->va) {
 354                         ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n");
 355                 } else {
 356                         dma_free_coherent(dev->hw->device, mem->size, mem->va,
 357                                           mem->pa);
 358                         mem->va = NULL;
 359                 }
 360         }
 361
 362         return ret_code;
 363 }
 364
 365 /**
 366  * irdma_sc_del_hmc_obj - remove pe hmc objects
 367  * @dev: pointer to the device structure
 368  * @info: pointer to irdma_hmc_del_obj_info struct
 369  * @reset: true if called before reset
 370  *
 371  * This will de-populate the SDs and PDs.  It frees
 372  * the memory for PDS and backing storage.  After this function is returned,
 373  * caller should deallocate memory allocated previously for
 374  * book-keeping information about PDs and backing storage.
 375  */
 376 enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
 377                                             struct irdma_hmc_del_obj_info *info,
 378                                             bool reset)
 379 {
 380         struct irdma_hmc_pd_table *pd_table;
 381         u32 sd_idx, sd_lmt;
 382         u32 pd_idx, pd_lmt, rel_pd_idx;
 383         u32 i, j;
 384         enum irdma_status_code ret_code = 0;
 385
 386         if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
 387                 ibdev_dbg(to_ibdev(dev),
 388                           "HMC: error start_idx[%04d]  >= [type %04d].cnt[%04d]\n",
 389                           info->start_idx, info->rsrc_type,
 390                           info->hmc_info->hmc_obj[info->rsrc_type].cnt);
 391                 return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
 392         }
 393
 394         if ((info->start_idx + info->count) >
 395             info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
 396                 ibdev_dbg(to_ibdev(dev),
 397                           "HMC: error start_idx[%04d] + count %04d  >= [type %04d].cnt[%04d]\n",
 398                           info->start_idx, info->count, info->rsrc_type,
 399                           info->hmc_info->hmc_obj[info->rsrc_type].cnt);
 400                 return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
 401         }
 402
 403         irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
 404                                   info->start_idx, info->count, &pd_idx,
 405                                   &pd_lmt);
 406
 407         for (j = pd_idx; j < pd_lmt; j++) {
 408                 sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;
 409
 410                 if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid)
 411                         continue;
 412
 413                 if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
 414                     IRDMA_SD_TYPE_PAGED)
 415                         continue;
 416
 417                 rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;
 418                 pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
 419                 if (pd_table->pd_entry &&
 420                     pd_table->pd_entry[rel_pd_idx].valid) {
 421                         ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j);
 422                         if (ret_code) {
 423                                 ibdev_dbg(to_ibdev(dev),
 424                                           "HMC: remove_pd_bp error\n");
 425                                 return ret_code;
 426                         }
 427                 }
 428         }
 429
 430         irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
 431                                   info->start_idx, info->count, &sd_idx,
 432                                   &sd_lmt);
 433         if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
 434             sd_lmt > info->hmc_info->sd_table.sd_cnt) {
 435                 ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");
 436                 return IRDMA_ERR_INVALID_SD_INDEX;
 437         }
 438
 439         for (i = sd_idx; i < sd_lmt; i++) {
 440                 pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table;
 441                 if (!info->hmc_info->sd_table.sd_entry[i].valid)
 442                         continue;
 443                 switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
 444                 case IRDMA_SD_TYPE_DIRECT:
 445                         ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i);
 446                         if (!ret_code) {
 447                                 info->hmc_info->sd_indexes[info->del_sd_cnt] =
 448                                         (u16)i;
 449                                 info->del_sd_cnt++;
 450                         }
 451                         break;
 452                 case IRDMA_SD_TYPE_PAGED:
 453                         ret_code = irdma_prep_remove_pd_page(info->hmc_info, i);
 454                         if (ret_code)
 455                                 break;
 456                         if (dev->hmc_info != info->hmc_info &&
 457                             info->rsrc_type == IRDMA_HMC_IW_PBLE &&
 458                             pd_table->pd_entry) {
 459                                 kfree(pd_table->pd_entry_virt_mem.va);
 460                                 pd_table->pd_entry = NULL;
 461                         }
 462                         info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
 463                         info->del_sd_cnt++;
 464                         break;
 465                 default:
 466                         break;
 467                 }
 468         }
 469         return irdma_finish_del_sd_reg(dev, info, reset);
 470 }
 471
 472 /**
 473  * irdma_add_sd_table_entry - Adds a segment descriptor to the table
 474  * @hw: pointer to our hw struct
 475  * @hmc_info: pointer to the HMC configuration information struct
 476  * @sd_index: segment descriptor index to manipulate
 477  * @type: what type of segment descriptor we're manipulating
 478  * @direct_mode_sz: size to alloc in direct mode
 479  */
 480 enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
 481                                                 struct irdma_hmc_info *hmc_info,
 482                                                 u32 sd_index,
 483                                                 enum irdma_sd_entry_type type,
 484                                                 u64 direct_mode_sz)
 485 {
 486         struct irdma_hmc_sd_entry *sd_entry;
 487         struct irdma_dma_mem dma_mem;
 488         u64 alloc_len;
 489
 490         sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
 491         if (!sd_entry->valid) {
 492                 if (type == IRDMA_SD_TYPE_PAGED)
 493                         alloc_len = IRDMA_HMC_PAGED_BP_SIZE;
 494                 else
 495                         alloc_len = direct_mode_sz;
 496
 497                 /* allocate a 4K pd page or 2M backing page */
 498                 dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
 499                 dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size,
 500                                                 &dma_mem.pa, GFP_KERNEL);
 501                 if (!dma_mem.va)
 502                         return IRDMA_ERR_NO_MEMORY;
 503                 if (type == IRDMA_SD_TYPE_PAGED) {
 504                         struct irdma_virt_mem *vmem =
 505                                 &sd_entry->u.pd_table.pd_entry_virt_mem;
 506
 507                         vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
 508                         vmem->va = kzalloc(vmem->size, GFP_KERNEL);
 509                         if (!vmem->va) {
 510                                 dma_free_coherent(hw->device, dma_mem.size,
 511                                                   dma_mem.va, dma_mem.pa);
 512                                 dma_mem.va = NULL;
 513                                 return IRDMA_ERR_NO_MEMORY;
 514                         }
 515                         sd_entry->u.pd_table.pd_entry = vmem->va;
 516
 517                         memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem,
 518                                sizeof(sd_entry->u.pd_table.pd_page_addr));
 519                 } else {
 520                         memcpy(&sd_entry->u.bp.addr, &dma_mem,
 521                                sizeof(sd_entry->u.bp.addr));
 522
 523                         sd_entry->u.bp.sd_pd_index = sd_index;
 524                 }
 525
 526                 hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
 527                 hmc_info->sd_table.use_cnt++;
 528         }
 529         if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT)
 530                 sd_entry->u.bp.use_cnt++;
 531
 532         return 0;
 533 }
 534
 535 /**
 536  * irdma_add_pd_table_entry - Adds page descriptor to the specified table
 537  * @dev: pointer to our device structure
 538  * @hmc_info: pointer to the HMC configuration information structure
 539  * @pd_index: which page descriptor index to manipulate
 540  * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
 541  *
 542  * This function:
 543  *      1. Initializes the pd entry
 544  *      2. Adds pd_entry in the pd_table
 545  *      3. Mark the entry valid in irdma_hmc_pd_entry structure
 546  *      4. Initializes the pd_entry's ref count to 1
 547  * assumptions:
 548  *      1. The memory for pd should be pinned down, physically contiguous and
 549  *         aligned on 4K boundary and zeroed memory.
 550  *      2. It should be 4K in size.
 551  */
 552 enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
 553                                                 struct irdma_hmc_info *hmc_info,
 554                                                 u32 pd_index,
 555                                                 struct irdma_dma_mem *rsrc_pg)
 556 {
 557         struct irdma_hmc_pd_table *pd_table;
 558         struct irdma_hmc_pd_entry *pd_entry;
 559         struct irdma_dma_mem mem;
 560         struct irdma_dma_mem *page = &mem;
 561         u32 sd_idx, rel_pd_idx;
 562         u64 *pd_addr;
 563         u64 page_desc;
 564
 565         if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
 566                 return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
 567
 568         sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
 569         if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
 570             IRDMA_SD_TYPE_PAGED)
 571                 return 0;
 572
 573         rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);
 574         pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
 575         pd_entry = &pd_table->pd_entry[rel_pd_idx];
 576         if (!pd_entry->valid) {
 577                 if (rsrc_pg) {
 578                         pd_entry->rsrc_pg = true;
 579                         page = rsrc_pg;
 580                 } else {
 581                         page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,
 582                                            IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
 583                         page->va = dma_alloc_coherent(dev->hw->device,
 584                                                       page->size, &page->pa,
 585                                                       GFP_KERNEL);
 586                         if (!page->va)
 587                                 return IRDMA_ERR_NO_MEMORY;
 588
 589                         pd_entry->rsrc_pg = false;
 590                 }
 591
 592                 memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
 593                 pd_entry->bp.sd_pd_index = pd_index;
 594                 pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED;
 595                 page_desc = page->pa | 0x1;
 596                 pd_addr = pd_table->pd_page_addr.va;
 597                 pd_addr += rel_pd_idx;
 598                 memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
 599                 pd_entry->sd_index = sd_idx;
 600                 pd_entry->valid = true;
 601                 pd_table->use_cnt++;
 602                 irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx);
 603         }
 604         pd_entry->bp.use_cnt++;
 605
 606         return 0;
 607 }
 608
 609 /**
 610  * irdma_remove_pd_bp - remove a backing page from a page descriptor
 611  * @dev: pointer to our HW structure
 612  * @hmc_info: pointer to the HMC configuration information structure
 613  * @idx: the page index
 614  *
 615  * This function:
 616  *      1. Marks the entry in pd table (for paged address mode) or in sd table
 617  *         (for direct address mode) invalid.
 618  *      2. Write to register PMPDINV to invalidate the backing page in FV cache
 619  *      3. Decrement the ref count for the pd _entry
 620  * assumptions:
 621  *      1. Caller can deallocate the memory used by backing storage after this
 622  *         function returns.
 623  */
 624 enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
 625                                           struct irdma_hmc_info *hmc_info,
 626                                           u32 idx)
 627 {
 628         struct irdma_hmc_pd_entry *pd_entry;
 629         struct irdma_hmc_pd_table *pd_table;
 630         struct irdma_hmc_sd_entry *sd_entry;
 631         u32 sd_idx, rel_pd_idx;
 632         struct irdma_dma_mem *mem;
 633         u64 *pd_addr;
 634
 635         sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
 636         rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
 637         if (sd_idx >= hmc_info->sd_table.sd_cnt)
 638                 return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
 639
 640         sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
 641         if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
 642                 return IRDMA_ERR_INVALID_SD_TYPE;
 643
 644         pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
 645         pd_entry = &pd_table->pd_entry[rel_pd_idx];
 646         if (--pd_entry->bp.use_cnt)
 647                 return 0;
 648
 649         pd_entry->valid = false;
 650         pd_table->use_cnt--;
 651         pd_addr = pd_table->pd_page_addr.va;
 652         pd_addr += rel_pd_idx;
 653         memset(pd_addr, 0, sizeof(u64));
 654         irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx);
 655
 656         if (!pd_entry->rsrc_pg) {
 657                 mem = &pd_entry->bp.addr;
 658                 if (!mem || !mem->va)
 659                         return IRDMA_ERR_PARAM;
 660
 661                 dma_free_coherent(dev->hw->device, mem->size, mem->va,
 662                                   mem->pa);
 663                 mem->va = NULL;
 664         }
 665         if (!pd_table->use_cnt)
 666                 kfree(pd_table->pd_entry_virt_mem.va);
 667
 668         return 0;
 669 }
 670
 671 /**
 672  * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
 673  * @hmc_info: pointer to the HMC configuration information structure
 674  * @idx: the page index
 675  */
 676 enum irdma_status_code irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info,
 677                                                u32 idx)
 678 {
 679         struct irdma_hmc_sd_entry *sd_entry;
 680
 681         sd_entry = &hmc_info->sd_table.sd_entry[idx];
 682         if (--sd_entry->u.bp.use_cnt)
 683                 return IRDMA_ERR_NOT_READY;
 684
 685         hmc_info->sd_table.use_cnt--;
 686         sd_entry->valid = false;
 687
 688         return 0;
 689 }
 690
 691 /**
 692  * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
 693  * @hmc_info: pointer to the HMC configuration information structure
 694  * @idx: segment descriptor index to find the relevant page descriptor
 695  */
 696 enum irdma_status_code
 697 irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
 698 {
 699         struct irdma_hmc_sd_entry *sd_entry;
 700
 701         sd_entry = &hmc_info->sd_table.sd_entry[idx];
 702
 703         if (sd_entry->u.pd_table.use_cnt)
 704                 return IRDMA_ERR_NOT_READY;
 705
 706         sd_entry->valid = false;
 707         hmc_info->sd_table.use_cnt--;
 708
 709         return 0;
 710 }