drivers/infiniband/hw/irdma/pble.c

   1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
   2 /* Copyright (c) 2015 - 2021 Intel Corporation */
   3 #include "osdep.h"
   4 #include "status.h"
   5 #include "hmc.h"
   6 #include "defs.h"
   7 #include "type.h"
   8 #include "protos.h"
   9 #include "pble.h"
  10
  11 static enum irdma_status_code
  12 add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
  13
  14 /**
  15  * irdma_destroy_pble_prm - destroy prm during module unload
  16  * @pble_rsrc: pble resources
  17  */
  18 void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
  19 {
  20         struct irdma_chunk *chunk;
  21         struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo;
  22
  23         while (!list_empty(&pinfo->clist)) {
  24                 chunk = (struct irdma_chunk *) pinfo->clist.next;
  25                 list_del(&chunk->list);
  26                 if (chunk->type == PBLE_SD_PAGED)
  27                         irdma_pble_free_paged_mem(chunk);
  28                 if (chunk->bitmapbuf)
  29                         kfree(chunk->bitmapmem.va);
  30                 kfree(chunk->chunkmem.va);
  31         }
  32 }
  33
  34 /**
  35  * irdma_hmc_init_pble - Initialize pble resources during module load
  36  * @dev: irdma_sc_dev struct
  37  * @pble_rsrc: pble resources
  38  */
  39 enum irdma_status_code
  40 irdma_hmc_init_pble(struct irdma_sc_dev *dev,
  41                     struct irdma_hmc_pble_rsrc *pble_rsrc)
  42 {
  43         struct irdma_hmc_info *hmc_info;
  44         u32 fpm_idx = 0;
  45         enum irdma_status_code status = 0;
  46
  47         hmc_info = dev->hmc_info;
  48         pble_rsrc->dev = dev;
  49         pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base;
  50         /* Start pble' on 4k boundary */
  51         if (pble_rsrc->fpm_base_addr & 0xfff)
  52                 fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
  53         pble_rsrc->unallocated_pble =
  54                 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx;
  55         pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
  56         pble_rsrc->pinfo.pble_shift = PBLE_SHIFT;
  57
  58         mutex_init(&pble_rsrc->pble_mutex_lock);
  59
  60         spin_lock_init(&pble_rsrc->pinfo.prm_lock);
  61         INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
  62         if (add_pble_prm(pble_rsrc)) {
  63                 irdma_destroy_pble_prm(pble_rsrc);
  64                 status = IRDMA_ERR_NO_MEMORY;
  65         }
  66
  67         return status;
  68 }
  69
  70 /**
  71  * get_sd_pd_idx -  Returns sd index, pd index and rel_pd_idx from fpm address
  72  * @pble_rsrc: structure containing fpm address
  73  * @idx: where to return indexes
  74  */
  75 static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc,
  76                           struct sd_pd_idx *idx)
  77 {
  78         idx->sd_idx = (u32)pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE;
  79         idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE);
  80         idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD);
  81 }
  82
  83 /**
  84  * add_sd_direct - add sd direct for pble
  85  * @pble_rsrc: pble resource ptr
  86  * @info: page info for sd
  87  */
  88 static enum irdma_status_code
  89 add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,
  90               struct irdma_add_page_info *info)
  91 {
  92         struct irdma_sc_dev *dev = pble_rsrc->dev;
  93         enum irdma_status_code ret_code = 0;
  94         struct sd_pd_idx *idx = &info->idx;
  95         struct irdma_chunk *chunk = info->chunk;
  96         struct irdma_hmc_info *hmc_info = info->hmc_info;
  97         struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
  98         u32 offset = 0;
  99
 100         if (!sd_entry->valid) {
 101                 ret_code = irdma_add_sd_table_entry(dev->hw, hmc_info,
 102                                                     info->idx.sd_idx,
 103                                                     IRDMA_SD_TYPE_DIRECT,
 104                                                     IRDMA_HMC_DIRECT_BP_SIZE);
 105                 if (ret_code)
 106                         return ret_code;
 107
 108                 chunk->type = PBLE_SD_CONTIGOUS;
 109         }
 110
 111         offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT;
 112         chunk->size = info->pages << HMC_PAGED_BP_SHIFT;
 113         chunk->vaddr = sd_entry->u.bp.addr.va + offset;
 114         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 115         ibdev_dbg(to_ibdev(dev),
 116                   "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%pK fpm_addr = %llx\n",
 117                   chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
 118
 119         return 0;
 120 }
 121
 122 /**
 123  * fpm_to_idx - given fpm address, get pble index
 124  * @pble_rsrc: pble resource management
 125  * @addr: fpm address for index
 126  */
 127 static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr)
 128 {
 129         u64 idx;
 130
 131         idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3;
 132
 133         return (u32)idx;
 134 }
 135
 136 /**
 137  * add_bp_pages - add backing pages for sd
 138  * @pble_rsrc: pble resource management
 139  * @info: page info for sd
 140  */
 141 static enum irdma_status_code
 142 add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
 143              struct irdma_add_page_info *info)
 144 {
 145         struct irdma_sc_dev *dev = pble_rsrc->dev;
 146         u8 *addr;
 147         struct irdma_dma_mem mem;
 148         struct irdma_hmc_pd_entry *pd_entry;
 149         struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
 150         struct irdma_hmc_info *hmc_info = info->hmc_info;
 151         struct irdma_chunk *chunk = info->chunk;
 152         enum irdma_status_code status = 0;
 153         u32 rel_pd_idx = info->idx.rel_pd_idx;
 154         u32 pd_idx = info->idx.pd_idx;
 155         u32 i;
 156
 157         if (irdma_pble_get_paged_mem(chunk, info->pages))
 158                 return IRDMA_ERR_NO_MEMORY;
 159
 160         status = irdma_add_sd_table_entry(dev->hw, hmc_info, info->idx.sd_idx,
 161                                           IRDMA_SD_TYPE_PAGED,
 162                                           IRDMA_HMC_DIRECT_BP_SIZE);
 163         if (status)
 164                 goto error;
 165
 166         addr = chunk->vaddr;
 167         for (i = 0; i < info->pages; i++) {
 168                 mem.pa = (u64)chunk->dmainfo.dmaaddrs[i];
 169                 mem.size = 4096;
 170                 mem.va = addr;
 171                 pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
 172                 if (!pd_entry->valid) {
 173                         status = irdma_add_pd_table_entry(dev, hmc_info,
 174                                                           pd_idx++, &mem);
 175                         if (status)
 176                                 goto error;
 177
 178                         addr += 4096;
 179                 }
 180         }
 181
 182         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 183         return 0;
 184
 185 error:
 186         irdma_pble_free_paged_mem(chunk);
 187
 188         return status;
 189 }
 190
 191 /**
 192  * irdma_get_type - add a sd entry type for sd
 193  * @dev: irdma_sc_dev struct
 194  * @idx: index of sd
 195  * @pages: pages in the sd
 196  */
 197 static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev,
 198                                                struct sd_pd_idx *idx, u32 pages)
 199 {
 200         enum irdma_sd_entry_type sd_entry_type;
 201
 202         sd_entry_type = !idx->rel_pd_idx && pages == IRDMA_HMC_PD_CNT_IN_SD ?
 203                         IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;
 204         return sd_entry_type;
 205 }
 206
 207 /**
 208  * add_pble_prm - add a sd entry for pble resoure
 209  * @pble_rsrc: pble resource management
 210  */
 211 static enum irdma_status_code
 212 add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
 213 {
 214         struct irdma_sc_dev *dev = pble_rsrc->dev;
 215         struct irdma_hmc_sd_entry *sd_entry;
 216         struct irdma_hmc_info *hmc_info;
 217         struct irdma_chunk *chunk;
 218         struct irdma_add_page_info info;
 219         struct sd_pd_idx *idx = &info.idx;
 220         enum irdma_status_code ret_code = 0;
 221         enum irdma_sd_entry_type sd_entry_type;
 222         u64 sd_reg_val = 0;
 223         struct irdma_virt_mem chunkmem;
 224         u32 pages;
 225
 226         if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
 227                 return IRDMA_ERR_NO_MEMORY;
 228
 229         if (pble_rsrc->next_fpm_addr & 0xfff)
 230                 return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
 231
 232         chunkmem.size = sizeof(*chunk);
 233         chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);
 234         if (!chunkmem.va)
 235                 return IRDMA_ERR_NO_MEMORY;
 236
 237         chunk = chunkmem.va;
 238         chunk->chunkmem = chunkmem;
 239         hmc_info = dev->hmc_info;
 240         chunk->dev = dev;
 241         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 242         get_sd_pd_idx(pble_rsrc, idx);
 243         sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
 244         pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) :
 245                                     IRDMA_HMC_PD_CNT_IN_SD;
 246         pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
 247         info.chunk = chunk;
 248         info.hmc_info = hmc_info;
 249         info.pages = pages;
 250         info.sd_entry = sd_entry;
 251         if (!sd_entry->valid)
 252                 sd_entry_type = irdma_get_type(dev, idx, pages);
 253         else
 254                 sd_entry_type = sd_entry->entry_type;
 255
 256         ibdev_dbg(to_ibdev(dev),
 257                   "PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n",
 258                   pages, pble_rsrc->unallocated_pble,
 259                   pble_rsrc->next_fpm_addr);
 260         ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n", sd_entry_type);
 261         if (sd_entry_type == IRDMA_SD_TYPE_DIRECT)
 262                 ret_code = add_sd_direct(pble_rsrc, &info);
 263
 264         if (ret_code)
 265                 sd_entry_type = IRDMA_SD_TYPE_PAGED;
 266         else
 267                 pble_rsrc->stats_direct_sds++;
 268
 269         if (sd_entry_type == IRDMA_SD_TYPE_PAGED) {
 270                 ret_code = add_bp_pages(pble_rsrc, &info);
 271                 if (ret_code)
 272                         goto error;
 273                 else
 274                         pble_rsrc->stats_paged_sds++;
 275         }
 276
 277         ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk);
 278         if (ret_code)
 279                 goto error;
 280
 281         pble_rsrc->next_fpm_addr += chunk->size;
 282         ibdev_dbg(to_ibdev(dev),
 283                   "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n",
 284                   pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
 285         pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3);
 286         list_add(&chunk->list, &pble_rsrc->pinfo.clist);
 287         sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ?
 288                              sd_entry->u.pd_table.pd_page_addr.pa :
 289                              sd_entry->u.bp.addr.pa;
 290
 291         if (!sd_entry->valid) {
 292                 ret_code = irdma_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val,
 293                                             idx->sd_idx, sd_entry->entry_type, true);
 294                 if (ret_code)
 295                         goto error;
 296         }
 297
 298         sd_entry->valid = true;
 299         return 0;
 300
 301 error:
 302         if (chunk->bitmapbuf)
 303                 kfree(chunk->bitmapmem.va);
 304         kfree(chunk->chunkmem.va);
 305
 306         return ret_code;
 307 }
 308
 309 /**
 310  * free_lvl2 - fee level 2 pble
 311  * @pble_rsrc: pble resource management
 312  * @palloc: level 2 pble allocation
 313  */
 314 static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
 315                       struct irdma_pble_alloc *palloc)
 316 {
 317         u32 i;
 318         struct irdma_pble_level2 *lvl2 = &palloc->level2;
 319         struct irdma_pble_info *root = &lvl2->root;
 320         struct irdma_pble_info *leaf = lvl2->leaf;
 321
 322         for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
 323                 if (leaf->addr)
 324                         irdma_prm_return_pbles(&pble_rsrc->pinfo,
 325                                                &leaf->chunkinfo);
 326                 else
 327                         break;
 328         }
 329
 330         if (root->addr)
 331                 irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo);
 332
 333         kfree(lvl2->leafmem.va);
 334         lvl2->leaf = NULL;
 335 }
 336
 337 /**
 338  * get_lvl2_pble - get level 2 pble resource
 339  * @pble_rsrc: pble resource management
 340  * @palloc: level 2 pble allocation
 341  */
 342 static enum irdma_status_code
 343 get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 344               struct irdma_pble_alloc *palloc)
 345 {
 346         u32 lf4k, lflast, total, i;
 347         u32 pblcnt = PBLE_PER_PAGE;
 348         u64 *addr;
 349         struct irdma_pble_level2 *lvl2 = &palloc->level2;
 350         struct irdma_pble_info *root = &lvl2->root;
 351         struct irdma_pble_info *leaf;
 352         enum irdma_status_code ret_code;
 353         u64 fpm_addr;
 354
 355         /* number of full 512 (4K) leafs) */
 356         lf4k = palloc->total_cnt >> 9;
 357         lflast = palloc->total_cnt % PBLE_PER_PAGE;
 358         total = (lflast == 0) ? lf4k : lf4k + 1;
 359         lvl2->leaf_cnt = total;
 360
 361         lvl2->leafmem.size = (sizeof(*leaf) * total);
 362         lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
 363         if (!lvl2->leafmem.va)
 364                 return IRDMA_ERR_NO_MEMORY;
 365
 366         lvl2->leaf = lvl2->leafmem.va;
 367         leaf = lvl2->leaf;
 368         ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo,
 369                                        total << 3, &root->addr, &fpm_addr);
 370         if (ret_code) {
 371                 kfree(lvl2->leafmem.va);
 372                 lvl2->leaf = NULL;
 373                 return IRDMA_ERR_NO_MEMORY;
 374         }
 375
 376         root->idx = fpm_to_idx(pble_rsrc, fpm_addr);
 377         root->cnt = total;
 378         addr = root->addr;
 379         for (i = 0; i < total; i++, leaf++) {
 380                 pblcnt = (lflast && ((i + 1) == total)) ?
 381                                 lflast : PBLE_PER_PAGE;
 382                 ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo,
 383                                                &leaf->chunkinfo, pblcnt << 3,
 384                                                &leaf->addr, &fpm_addr);
 385                 if (ret_code)
 386                         goto error;
 387
 388                 leaf->idx = fpm_to_idx(pble_rsrc, fpm_addr);
 389
 390                 leaf->cnt = pblcnt;
 391                 *addr = (u64)leaf->idx;
 392                 addr++;
 393         }
 394
 395         palloc->level = PBLE_LEVEL_2;
 396         pble_rsrc->stats_lvl2++;
 397         return 0;
 398
 399 error:
 400         free_lvl2(pble_rsrc, palloc);
 401
 402         return IRDMA_ERR_NO_MEMORY;
 403 }
 404
 405 /**
 406  * get_lvl1_pble - get level 1 pble resource
 407  * @pble_rsrc: pble resource management
 408  * @palloc: level 1 pble allocation
 409  */
 410 static enum irdma_status_code
 411 get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 412               struct irdma_pble_alloc *palloc)
 413 {
 414         enum irdma_status_code ret_code;
 415         u64 fpm_addr;
 416         struct irdma_pble_info *lvl1 = &palloc->level1;
 417
 418         ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &lvl1->chunkinfo,
 419                                        palloc->total_cnt << 3, &lvl1->addr,
 420                                        &fpm_addr);
 421         if (ret_code)
 422                 return IRDMA_ERR_NO_MEMORY;
 423
 424         palloc->level = PBLE_LEVEL_1;
 425         lvl1->idx = fpm_to_idx(pble_rsrc, fpm_addr);
 426         lvl1->cnt = palloc->total_cnt;
 427         pble_rsrc->stats_lvl1++;
 428
 429         return 0;
 430 }
 431
 432 /**
 433  * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
 434  * @pble_rsrc: pble resources
 435  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
 436  * @level1_only: flag for a level 1 PBLE
 437  */
 438 static enum irdma_status_code
 439 get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 440                    struct irdma_pble_alloc *palloc, bool level1_only)
 441 {
 442         enum irdma_status_code status = 0;
 443
 444         status = get_lvl1_pble(pble_rsrc, palloc);
 445         if (!status || level1_only || palloc->total_cnt <= PBLE_PER_PAGE)
 446                 return status;
 447
 448         status = get_lvl2_pble(pble_rsrc, palloc);
 449
 450         return status;
 451 }
 452
 453 /**
 454  * irdma_get_pble - allocate pbles from the prm
 455  * @pble_rsrc: pble resources
 456  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
 457  * @pble_cnt: #of pbles requested
 458  * @level1_only: true if only pble level 1 to acquire
 459  */
 460 enum irdma_status_code irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 461                                       struct irdma_pble_alloc *palloc,
 462                                       u32 pble_cnt, bool level1_only)
 463 {
 464         enum irdma_status_code status = 0;
 465         int max_sds = 0;
 466         int i;
 467
 468         palloc->total_cnt = pble_cnt;
 469         palloc->level = PBLE_LEVEL_0;
 470
 471         mutex_lock(&pble_rsrc->pble_mutex_lock);
 472
 473         /*check first to see if we can get pble's without acquiring
 474          * additional sd's
 475          */
 476         status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only);
 477         if (!status)
 478                 goto exit;
 479
 480         max_sds = (palloc->total_cnt >> 18) + 1;
 481         for (i = 0; i < max_sds; i++) {
 482                 status = add_pble_prm(pble_rsrc);
 483                 if (status)
 484                         break;
 485
 486                 status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only);
 487                 /* if level1_only, only go through it once */
 488                 if (!status || level1_only)
 489                         break;
 490         }
 491
 492 exit:
 493         if (!status) {
 494                 pble_rsrc->allocdpbles += pble_cnt;
 495                 pble_rsrc->stats_alloc_ok++;
 496         } else {
 497                 pble_rsrc->stats_alloc_fail++;
 498         }
 499         mutex_unlock(&pble_rsrc->pble_mutex_lock);
 500
 501         return status;
 502 }
 503
 504 /**
 505  * irdma_free_pble - put pbles back into prm
 506  * @pble_rsrc: pble resources
 507  * @palloc: contains all information regarding pble resource being freed
 508  */
 509 void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 510                      struct irdma_pble_alloc *palloc)
 511 {
 512         pble_rsrc->freedpbles += palloc->total_cnt;
 513
 514         if (palloc->level == PBLE_LEVEL_2)
 515                 free_lvl2(pble_rsrc, palloc);
 516         else
 517                 irdma_prm_return_pbles(&pble_rsrc->pinfo,
 518                                        &palloc->level1.chunkinfo);
 519         pble_rsrc->stats_alloc_freed++;
 520 }