drivers/misc/habanalabs/common/command_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/mm.h>
  12 #include <linux/slab.h>
  13 #include <linux/uaccess.h>
  14
  15 static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
  16 {
  17         struct hl_device *hdev = ctx->hdev;
  18         struct asic_fixed_properties *prop = &hdev->asic_prop;
  19         struct hl_vm_va_block *va_block, *tmp;
  20         dma_addr_t bus_addr;
  21         u64 virt_addr;
  22         u32 page_size = prop->pmmu.page_size;
  23         s32 offset;
  24         int rc;
  25
  26         if (!hdev->supports_cb_mapping) {
  27                 dev_err_ratelimited(hdev->dev,
  28                                 "Cannot map CB because no VA range is allocated for CB mapping\n");
  29                 return -EINVAL;
  30         }
  31
  32         if (!hdev->mmu_enable) {
  33                 dev_err_ratelimited(hdev->dev,
  34                                 "Cannot map CB because MMU is disabled\n");
  35                 return -EINVAL;
  36         }
  37
  38         INIT_LIST_HEAD(&cb->va_block_list);
  39
  40         for (bus_addr = cb->bus_address;
  41                         bus_addr < cb->bus_address + cb->size;
  42                         bus_addr += page_size) {
  43
  44                 virt_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, page_size);
  45                 if (!virt_addr) {
  46                         dev_err(hdev->dev,
  47                                 "Failed to allocate device virtual address for CB\n");
  48                         rc = -ENOMEM;
  49                         goto err_va_pool_free;
  50                 }
  51
  52                 va_block = kzalloc(sizeof(*va_block), GFP_KERNEL);
  53                 if (!va_block) {
  54                         rc = -ENOMEM;
  55                         gen_pool_free(ctx->cb_va_pool, virt_addr, page_size);
  56                         goto err_va_pool_free;
  57                 }
  58
  59                 va_block->start = virt_addr;
  60                 va_block->end = virt_addr + page_size;
  61                 va_block->size = page_size;
  62                 list_add_tail(&va_block->node, &cb->va_block_list);
  63         }
  64
  65         mutex_lock(&ctx->mmu_lock);
  66
  67         bus_addr = cb->bus_address;
  68         offset = 0;
  69         list_for_each_entry(va_block, &cb->va_block_list, node) {
  70                 rc = hl_mmu_map_page(ctx, va_block->start, bus_addr,
  71                                 va_block->size, list_is_last(&va_block->node,
  72                                                         &cb->va_block_list));
  73                 if (rc) {
  74                         dev_err(hdev->dev, "Failed to map VA %#llx to CB\n",
  75                                 va_block->start);
  76                         goto err_va_umap;
  77                 }
  78
  79                 bus_addr += va_block->size;
  80                 offset += va_block->size;
  81         }
  82
  83         hdev->asic_funcs->mmu_invalidate_cache(hdev, false, VM_TYPE_USERPTR);
  84
  85         mutex_unlock(&ctx->mmu_lock);
  86
  87         cb->is_mmu_mapped = true;
  88
  89         return 0;
  90
  91 err_va_umap:
  92         list_for_each_entry(va_block, &cb->va_block_list, node) {
  93                 if (offset <= 0)
  94                         break;
  95                 hl_mmu_unmap_page(ctx, va_block->start, va_block->size,
  96                                 offset <= va_block->size);
  97                 offset -= va_block->size;
  98         }
  99
 100         hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
 101
 102         mutex_unlock(&ctx->mmu_lock);
 103
 104 err_va_pool_free:
 105         list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) {
 106                 gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size);
 107                 list_del(&va_block->node);
 108                 kfree(va_block);
 109         }
 110
 111         return rc;
 112 }
 113
 114 static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)
 115 {
 116         struct hl_device *hdev = ctx->hdev;
 117         struct hl_vm_va_block *va_block, *tmp;
 118
 119         mutex_lock(&ctx->mmu_lock);
 120
 121         list_for_each_entry(va_block, &cb->va_block_list, node)
 122                 if (hl_mmu_unmap_page(ctx, va_block->start, va_block->size,
 123                                 list_is_last(&va_block->node,
 124                                                 &cb->va_block_list)))
 125                         dev_warn_ratelimited(hdev->dev,
 126                                         "Failed to unmap CB's va 0x%llx\n",
 127                                         va_block->start);
 128
 129         hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
 130
 131         mutex_unlock(&ctx->mmu_lock);
 132
 133         list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) {
 134                 gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size);
 135                 list_del(&va_block->node);
 136                 kfree(va_block);
 137         }
 138 }
 139
 140 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
 141 {
 142         if (cb->is_internal)
 143                 gen_pool_free(hdev->internal_cb_pool,
 144                                 (uintptr_t)cb->kernel_address, cb->size);
 145         else
 146                 hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
 147                                 cb->kernel_address, cb->bus_address);
 148
 149         kfree(cb);
 150 }
 151
 152 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
 153 {
 154         if (cb->is_pool) {
 155                 spin_lock(&hdev->cb_pool_lock);
 156                 list_add(&cb->pool_list, &hdev->cb_pool);
 157                 spin_unlock(&hdev->cb_pool_lock);
 158         } else {
 159                 cb_fini(hdev, cb);
 160         }
 161 }
 162
 163 static void cb_release(struct kref *ref)
 164 {
 165         struct hl_device *hdev;
 166         struct hl_cb *cb;
 167
 168         cb = container_of(ref, struct hl_cb, refcount);
 169         hdev = cb->hdev;
 170
 171         hl_debugfs_remove_cb(cb);
 172
 173         if (cb->is_mmu_mapped)
 174                 cb_unmap_mem(cb->ctx, cb);
 175
 176         hl_ctx_put(cb->ctx);
 177
 178         cb_do_release(hdev, cb);
 179 }
 180
 181 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
 182                                         int ctx_id, bool internal_cb)
 183 {
 184         struct hl_cb *cb = NULL;
 185         u32 cb_offset;
 186         void *p;
 187
 188         /*
 189          * We use of GFP_ATOMIC here because this function can be called from
 190          * the latency-sensitive code path for command submission. Due to H/W
 191          * limitations in some of the ASICs, the kernel must copy the user CB
 192          * that is designated for an external queue and actually enqueue
 193          * the kernel's copy. Hence, we must never sleep in this code section
 194          * and must use GFP_ATOMIC for all memory allocations.
 195          */
 196         if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled)
 197                 cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
 198
 199         if (!cb)
 200                 cb = kzalloc(sizeof(*cb), GFP_KERNEL);
 201
 202         if (!cb)
 203                 return NULL;
 204
 205         if (internal_cb) {
 206                 p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size);
 207                 if (!p) {
 208                         kfree(cb);
 209                         return NULL;
 210                 }
 211
 212                 cb_offset = p - hdev->internal_cb_pool_virt_addr;
 213                 cb->is_internal = true;
 214                 cb->bus_address =  hdev->internal_cb_va_base + cb_offset;
 215         } else if (ctx_id == HL_KERNEL_ASID_ID) {
 216                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
 217                                                 &cb->bus_address, GFP_ATOMIC);
 218                 if (!p)
 219                         p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 220                                         cb_size, &cb->bus_address, GFP_KERNEL);
 221         } else {
 222                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
 223                                                 &cb->bus_address,
 224                                                 GFP_USER | __GFP_ZERO);
 225         }
 226
 227         if (!p) {
 228                 dev_err(hdev->dev,
 229                         "failed to allocate %d of dma memory for CB\n",
 230                         cb_size);
 231                 kfree(cb);
 232                 return NULL;
 233         }
 234
 235         cb->kernel_address = p;
 236         cb->size = cb_size;
 237
 238         return cb;
 239 }
 240
 241 int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 242                         struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
 243                         bool map_cb, u64 *handle)
 244 {
 245         struct hl_cb *cb;
 246         bool alloc_new_cb = true;
 247         int rc, ctx_id = ctx->asid;
 248
 249         /*
 250          * Can't use generic function to check this because of special case
 251          * where we create a CB as part of the reset process
 252          */
 253         if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
 254                                         (ctx_id != HL_KERNEL_ASID_ID))) {
 255                 dev_warn_ratelimited(hdev->dev,
 256                         "Device is disabled or in reset. Can't create new CBs\n");
 257                 rc = -EBUSY;
 258                 goto out_err;
 259         }
 260
 261         if (cb_size > SZ_2M) {
 262                 dev_err(hdev->dev, "CB size %d must be less than %d\n",
 263                         cb_size, SZ_2M);
 264                 rc = -EINVAL;
 265                 goto out_err;
 266         }
 267
 268         if (!internal_cb) {
 269                 /* Minimum allocation must be PAGE SIZE */
 270                 if (cb_size < PAGE_SIZE)
 271                         cb_size = PAGE_SIZE;
 272
 273                 if (ctx_id == HL_KERNEL_ASID_ID &&
 274                                 cb_size <= hdev->asic_prop.cb_pool_cb_size) {
 275
 276                         spin_lock(&hdev->cb_pool_lock);
 277                         if (!list_empty(&hdev->cb_pool)) {
 278                                 cb = list_first_entry(&hdev->cb_pool,
 279                                                 typeof(*cb), pool_list);
 280                                 list_del(&cb->pool_list);
 281                                 spin_unlock(&hdev->cb_pool_lock);
 282                                 alloc_new_cb = false;
 283                         } else {
 284                                 spin_unlock(&hdev->cb_pool_lock);
 285                                 dev_dbg(hdev->dev, "CB pool is empty\n");
 286                         }
 287                 }
 288         }
 289
 290         if (alloc_new_cb) {
 291                 cb = hl_cb_alloc(hdev, cb_size, ctx_id, internal_cb);
 292                 if (!cb) {
 293                         rc = -ENOMEM;
 294                         goto out_err;
 295                 }
 296         }
 297
 298         cb->hdev = hdev;
 299         cb->ctx = ctx;
 300         hl_ctx_get(hdev, cb->ctx);
 301
 302         if (map_cb) {
 303                 if (ctx_id == HL_KERNEL_ASID_ID) {
 304                         dev_err(hdev->dev,
 305                                 "CB mapping is not supported for kernel context\n");
 306                         rc = -EINVAL;
 307                         goto release_cb;
 308                 }
 309
 310                 rc = cb_map_mem(ctx, cb);
 311                 if (rc)
 312                         goto release_cb;
 313         }
 314
 315         spin_lock(&mgr->cb_lock);
 316         rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
 317         if (rc < 0)
 318                 rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_KERNEL);
 319         spin_unlock(&mgr->cb_lock);
 320
 321         if (rc < 0) {
 322                 dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
 323                 goto unmap_mem;
 324         }
 325
 326         cb->id = (u64) rc;
 327
 328         kref_init(&cb->refcount);
 329         spin_lock_init(&cb->lock);
 330
 331         /*
 332          * idr is 32-bit so we can safely OR it with a mask that is above
 333          * 32 bit
 334          */
 335         *handle = cb->id | HL_MMAP_TYPE_CB;
 336         *handle <<= PAGE_SHIFT;
 337
 338         hl_debugfs_add_cb(cb);
 339
 340         return 0;
 341
 342 unmap_mem:
 343         if (cb->is_mmu_mapped)
 344                 cb_unmap_mem(cb->ctx, cb);
 345 release_cb:
 346         hl_ctx_put(cb->ctx);
 347         cb_do_release(hdev, cb);
 348 out_err:
 349         *handle = 0;
 350
 351         return rc;
 352 }
 353
 354 int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
 355 {
 356         struct hl_cb *cb;
 357         u32 handle;
 358         int rc = 0;
 359
 360         /*
 361          * handle was given to user to do mmap, I need to shift it back to
 362          * how the idr module gave it to me
 363          */
 364         cb_handle >>= PAGE_SHIFT;
 365         handle = (u32) cb_handle;
 366
 367         spin_lock(&mgr->cb_lock);
 368
 369         cb = idr_find(&mgr->cb_handles, handle);
 370         if (cb) {
 371                 idr_remove(&mgr->cb_handles, handle);
 372                 spin_unlock(&mgr->cb_lock);
 373                 kref_put(&cb->refcount, cb_release);
 374         } else {
 375                 spin_unlock(&mgr->cb_lock);
 376                 dev_err(hdev->dev,
 377                         "CB destroy failed, no match to handle 0x%x\n", handle);
 378                 rc = -EINVAL;
 379         }
 380
 381         return rc;
 382 }
 383
 384 static int hl_cb_info(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 385                         u64 cb_handle, u32 *usage_cnt)
 386 {
 387         struct hl_cb *cb;
 388         u32 handle;
 389         int rc = 0;
 390
 391         /* The CB handle was given to user to do mmap, so need to shift it back
 392          * to the value which was allocated by the IDR module.
 393          */
 394         cb_handle >>= PAGE_SHIFT;
 395         handle = (u32) cb_handle;
 396
 397         spin_lock(&mgr->cb_lock);
 398
 399         cb = idr_find(&mgr->cb_handles, handle);
 400         if (!cb) {
 401                 dev_err(hdev->dev,
 402                         "CB info failed, no match to handle 0x%x\n", handle);
 403                 rc = -EINVAL;
 404                 goto out;
 405         }
 406
 407         *usage_cnt = atomic_read(&cb->cs_cnt);
 408
 409 out:
 410         spin_unlock(&mgr->cb_lock);
 411         return rc;
 412 }
 413
 414 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 415 {
 416         union hl_cb_args *args = data;
 417         struct hl_device *hdev = hpriv->hdev;
 418         enum hl_device_status status;
 419         u64 handle = 0;
 420         u32 usage_cnt = 0;
 421         int rc;
 422
 423         if (!hl_device_operational(hdev, &status)) {
 424                 dev_warn_ratelimited(hdev->dev,
 425                         "Device is %s. Can't execute CB IOCTL\n",
 426                         hdev->status[status]);
 427                 return -EBUSY;
 428         }
 429
 430         switch (args->in.op) {
 431         case HL_CB_OP_CREATE:
 432                 if (args->in.cb_size > HL_MAX_CB_SIZE) {
 433                         dev_err(hdev->dev,
 434                                 "User requested CB size %d must be less than %d\n",
 435                                 args->in.cb_size, HL_MAX_CB_SIZE);
 436                         rc = -EINVAL;
 437                 } else {
 438                         rc = hl_cb_create(hdev, &hpriv->cb_mgr, hpriv->ctx,
 439                                         args->in.cb_size, false,
 440                                         !!(args->in.flags & HL_CB_FLAGS_MAP),
 441                                         &handle);
 442                 }
 443
 444                 memset(args, 0, sizeof(*args));
 445                 args->out.cb_handle = handle;
 446                 break;
 447
 448         case HL_CB_OP_DESTROY:
 449                 rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
 450                                         args->in.cb_handle);
 451                 break;
 452
 453         case HL_CB_OP_INFO:
 454                 rc = hl_cb_info(hdev, &hpriv->cb_mgr, args->in.cb_handle,
 455                                 &usage_cnt);
 456                 memset(args, 0, sizeof(*args));
 457                 args->out.usage_cnt = usage_cnt;
 458                 break;
 459
 460         default:
 461                 rc = -ENOTTY;
 462                 break;
 463         }
 464
 465         return rc;
 466 }
 467
 468 static void cb_vm_close(struct vm_area_struct *vma)
 469 {
 470         struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
 471         long new_mmap_size;
 472
 473         new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
 474
 475         if (new_mmap_size > 0) {
 476                 cb->mmap_size = new_mmap_size;
 477                 return;
 478         }
 479
 480         spin_lock(&cb->lock);
 481         cb->mmap = false;
 482         spin_unlock(&cb->lock);
 483
 484         hl_cb_put(cb);
 485         vma->vm_private_data = NULL;
 486 }
 487
 488 static const struct vm_operations_struct cb_vm_ops = {
 489         .close = cb_vm_close
 490 };
 491
 492 int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 493 {
 494         struct hl_device *hdev = hpriv->hdev;
 495         struct hl_cb *cb;
 496         u32 handle, user_cb_size;
 497         int rc;
 498
 499         /* We use the page offset to hold the idr and thus we need to clear
 500          * it before doing the mmap itself
 501          */
 502         handle = vma->vm_pgoff;
 503         vma->vm_pgoff = 0;
 504
 505         /* reference was taken here */
 506         cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
 507         if (!cb) {
 508                 dev_err(hdev->dev,
 509                         "CB mmap failed, no match to handle 0x%x\n", handle);
 510                 return -EINVAL;
 511         }
 512
 513         /* Validation check */
 514         user_cb_size = vma->vm_end - vma->vm_start;
 515         if (user_cb_size != ALIGN(cb->size, PAGE_SIZE)) {
 516                 dev_err(hdev->dev,
 517                         "CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
 518                         vma->vm_end - vma->vm_start, cb->size);
 519                 rc = -EINVAL;
 520                 goto put_cb;
 521         }
 522
 523         if (!access_ok((void __user *) (uintptr_t) vma->vm_start,
 524                                                         user_cb_size)) {
 525                 dev_err(hdev->dev,
 526                         "user pointer is invalid - 0x%lx\n",
 527                         vma->vm_start);
 528
 529                 rc = -EINVAL;
 530                 goto put_cb;
 531         }
 532
 533         spin_lock(&cb->lock);
 534
 535         if (cb->mmap) {
 536                 dev_err(hdev->dev,
 537                         "CB mmap failed, CB already mmaped to user\n");
 538                 rc = -EINVAL;
 539                 goto release_lock;
 540         }
 541
 542         cb->mmap = true;
 543
 544         spin_unlock(&cb->lock);
 545
 546         vma->vm_ops = &cb_vm_ops;
 547
 548         /*
 549          * Note: We're transferring the cb reference to
 550          * vma->vm_private_data here.
 551          */
 552
 553         vma->vm_private_data = cb;
 554
 555         rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
 556                                         cb->bus_address, cb->size);
 557         if (rc) {
 558                 spin_lock(&cb->lock);
 559                 cb->mmap = false;
 560                 goto release_lock;
 561         }
 562
 563         cb->mmap_size = cb->size;
 564         vma->vm_pgoff = handle;
 565
 566         return 0;
 567
 568 release_lock:
 569         spin_unlock(&cb->lock);
 570 put_cb:
 571         hl_cb_put(cb);
 572         return rc;
 573 }
 574
 575 struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 576                         u32 handle)
 577 {
 578         struct hl_cb *cb;
 579
 580         spin_lock(&mgr->cb_lock);
 581         cb = idr_find(&mgr->cb_handles, handle);
 582
 583         if (!cb) {
 584                 spin_unlock(&mgr->cb_lock);
 585                 dev_warn(hdev->dev,
 586                         "CB get failed, no match to handle 0x%x\n", handle);
 587                 return NULL;
 588         }
 589
 590         kref_get(&cb->refcount);
 591
 592         spin_unlock(&mgr->cb_lock);
 593
 594         return cb;
 595
 596 }
 597
 598 void hl_cb_put(struct hl_cb *cb)
 599 {
 600         kref_put(&cb->refcount, cb_release);
 601 }
 602
 603 void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
 604 {
 605         spin_lock_init(&mgr->cb_lock);
 606         idr_init(&mgr->cb_handles);
 607 }
 608
 609 void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
 610 {
 611         struct hl_cb *cb;
 612         struct idr *idp;
 613         u32 id;
 614
 615         idp = &mgr->cb_handles;
 616
 617         idr_for_each_entry(idp, cb, id) {
 618                 if (kref_put(&cb->refcount, cb_release) != 1)
 619                         dev_err(hdev->dev,
 620                                 "CB %d for CTX ID %d is still alive\n",
 621                                 id, cb->ctx->asid);
 622         }
 623
 624         idr_destroy(&mgr->cb_handles);
 625 }
 626
 627 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
 628                                         bool internal_cb)
 629 {
 630         u64 cb_handle;
 631         struct hl_cb *cb;
 632         int rc;
 633
 634         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, cb_size,
 635                                 internal_cb, false, &cb_handle);
 636         if (rc) {
 637                 dev_err(hdev->dev,
 638                         "Failed to allocate CB for the kernel driver %d\n", rc);
 639                 return NULL;
 640         }
 641
 642         cb_handle >>= PAGE_SHIFT;
 643         cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
 644         /* hl_cb_get should never fail here */
 645         if (!cb) {
 646                 dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n",
 647                                 (u32) cb_handle);
 648                 goto destroy_cb;
 649         }
 650
 651         return cb;
 652
 653 destroy_cb:
 654         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
 655
 656         return NULL;
 657 }
 658
 659 int hl_cb_pool_init(struct hl_device *hdev)
 660 {
 661         struct hl_cb *cb;
 662         int i;
 663
 664         INIT_LIST_HEAD(&hdev->cb_pool);
 665         spin_lock_init(&hdev->cb_pool_lock);
 666
 667         for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
 668                 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
 669                                 HL_KERNEL_ASID_ID, false);
 670                 if (cb) {
 671                         cb->is_pool = true;
 672                         list_add(&cb->pool_list, &hdev->cb_pool);
 673                 } else {
 674                         hl_cb_pool_fini(hdev);
 675                         return -ENOMEM;
 676                 }
 677         }
 678
 679         return 0;
 680 }
 681
 682 int hl_cb_pool_fini(struct hl_device *hdev)
 683 {
 684         struct hl_cb *cb, *tmp;
 685
 686         list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
 687                 list_del(&cb->pool_list);
 688                 cb_fini(hdev, cb);
 689         }
 690
 691         return 0;
 692 }
 693
 694 int hl_cb_va_pool_init(struct hl_ctx *ctx)
 695 {
 696         struct hl_device *hdev = ctx->hdev;
 697         struct asic_fixed_properties *prop = &hdev->asic_prop;
 698         int rc;
 699
 700         if (!hdev->supports_cb_mapping)
 701                 return 0;
 702
 703         ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1);
 704         if (!ctx->cb_va_pool) {
 705                 dev_err(hdev->dev,
 706                         "Failed to create VA gen pool for CB mapping\n");
 707                 return -ENOMEM;
 708         }
 709
 710         rc = gen_pool_add(ctx->cb_va_pool, prop->cb_va_start_addr,
 711                         prop->cb_va_end_addr - prop->cb_va_start_addr, -1);
 712         if (rc) {
 713                 dev_err(hdev->dev,
 714                         "Failed to add memory to VA gen pool for CB mapping\n");
 715                 goto err_pool_destroy;
 716         }
 717
 718         return 0;
 719
 720 err_pool_destroy:
 721         gen_pool_destroy(ctx->cb_va_pool);
 722
 723         return rc;
 724 }
 725
 726 void hl_cb_va_pool_fini(struct hl_ctx *ctx)
 727 {
 728         struct hl_device *hdev = ctx->hdev;
 729
 730         if (!hdev->supports_cb_mapping)
 731                 return;
 732
 733         gen_pool_destroy(ctx->cb_va_pool);
 734 }