drivers/misc/habanalabs/common/memory.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10 #include "../include/hw_ip/mmu/mmu_general.h"
  11
  12 #include <linux/uaccess.h>
  13 #include <linux/slab.h>
  14
  15 #define HL_MMU_DEBUG    0
  16
  17 /*
  18  * The va ranges in context object contain a list with the available chunks of
  19  * device virtual memory.
  20  * There is one range for host allocations and one for DRAM allocations.
  21  *
  22  * On initialization each range contains one chunk of all of its available
  23  * virtual range which is a half of the total device virtual range.
  24  *
  25  * On each mapping of physical pages, a suitable virtual range chunk (with a
  26  * minimum size) is selected from the list. If the chunk size equals the
  27  * requested size, the chunk is returned. Otherwise, the chunk is split into
  28  * two chunks - one to return as result and a remainder to stay in the list.
  29  *
  30  * On each Unmapping of a virtual address, the relevant virtual chunk is
  31  * returned to the list. The chunk is added to the list and if its edges match
  32  * the edges of the adjacent chunks (means a contiguous chunk can be created),
  33  * the chunks are merged.
  34  *
  35  * On finish, the list is checked to have only one chunk of all the relevant
  36  * virtual range (which is a half of the device total virtual range).
  37  * If not (means not all mappings were unmapped), a warning is printed.
  38  */
  39
  40 /*
  41  * alloc_device_memory - allocate device memory
  42  *
  43  * @ctx                 : current context
  44  * @args                : host parameters containing the requested size
  45  * @ret_handle          : result handle
  46  *
  47  * This function does the following:
  48  * - Allocate the requested size rounded up to 'dram_page_size' pages
  49  * - Return unique handle
  50  */
  51 static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
  52                                 u32 *ret_handle)
  53 {
  54         struct hl_device *hdev = ctx->hdev;
  55         struct hl_vm *vm = &hdev->vm;
  56         struct hl_vm_phys_pg_pack *phys_pg_pack;
  57         u64 paddr = 0, total_size, num_pgs, i;
  58         u32 num_curr_pgs, page_size, page_shift;
  59         int handle, rc;
  60         bool contiguous;
  61
  62         num_curr_pgs = 0;
  63         page_size = hdev->asic_prop.dram_page_size;
  64         page_shift = __ffs(page_size);
  65         num_pgs = (args->alloc.mem_size + (page_size - 1)) >> page_shift;
  66         total_size = num_pgs << page_shift;
  67
  68         if (!total_size) {
  69                 dev_err(hdev->dev, "Cannot allocate 0 bytes\n");
  70                 return -EINVAL;
  71         }
  72
  73         contiguous = args->flags & HL_MEM_CONTIGUOUS;
  74
  75         if (contiguous) {
  76                 paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size);
  77                 if (!paddr) {
  78                         dev_err(hdev->dev,
  79                                 "failed to allocate %llu contiguous pages with total size of %llu\n",
  80                                 num_pgs, total_size);
  81                         return -ENOMEM;
  82                 }
  83
  84                 if (hdev->memory_scrub) {
  85                         rc = hdev->asic_funcs->scrub_device_mem(hdev, paddr,
  86                                         total_size);
  87                         if (rc) {
  88                                 dev_err(hdev->dev,
  89                                         "Failed to scrub contiguous device memory\n");
  90                                 goto pages_pack_err;
  91                         }
  92                 }
  93         }
  94
  95         phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
  96         if (!phys_pg_pack) {
  97                 rc = -ENOMEM;
  98                 goto pages_pack_err;
  99         }
 100
 101         phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;
 102         phys_pg_pack->asid = ctx->asid;
 103         phys_pg_pack->npages = num_pgs;
 104         phys_pg_pack->page_size = page_size;
 105         phys_pg_pack->total_size = total_size;
 106         phys_pg_pack->flags = args->flags;
 107         phys_pg_pack->contiguous = contiguous;
 108
 109         phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);
 110         if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
 111                 rc = -ENOMEM;
 112                 goto pages_arr_err;
 113         }
 114
 115         if (phys_pg_pack->contiguous) {
 116                 for (i = 0 ; i < num_pgs ; i++)
 117                         phys_pg_pack->pages[i] = paddr + i * page_size;
 118         } else {
 119                 for (i = 0 ; i < num_pgs ; i++) {
 120                         phys_pg_pack->pages[i] = (u64) gen_pool_alloc(
 121                                                         vm->dram_pg_pool,
 122                                                         page_size);
 123                         if (!phys_pg_pack->pages[i]) {
 124                                 dev_err(hdev->dev,
 125                                         "Failed to allocate device memory (out of memory)\n");
 126                                 rc = -ENOMEM;
 127                                 goto page_err;
 128                         }
 129
 130                         if (hdev->memory_scrub) {
 131                                 rc = hdev->asic_funcs->scrub_device_mem(hdev,
 132                                                 phys_pg_pack->pages[i],
 133                                                 page_size);
 134                                 if (rc) {
 135                                         dev_err(hdev->dev,
 136                                                 "Failed to scrub device memory\n");
 137                                         goto page_err;
 138                                 }
 139                         }
 140
 141                         num_curr_pgs++;
 142                 }
 143         }
 144
 145         spin_lock(&vm->idr_lock);
 146         handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
 147                                 GFP_ATOMIC);
 148         spin_unlock(&vm->idr_lock);
 149
 150         if (handle < 0) {
 151                 dev_err(hdev->dev, "Failed to get handle for page\n");
 152                 rc = -EFAULT;
 153                 goto idr_err;
 154         }
 155
 156         for (i = 0 ; i < num_pgs ; i++)
 157                 kref_get(&vm->dram_pg_pool_refcount);
 158
 159         phys_pg_pack->handle = handle;
 160
 161         atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);
 162         atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);
 163
 164         *ret_handle = handle;
 165
 166         return 0;
 167
 168 idr_err:
 169 page_err:
 170         if (!phys_pg_pack->contiguous)
 171                 for (i = 0 ; i < num_curr_pgs ; i++)
 172                         gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],
 173                                         page_size);
 174
 175         kvfree(phys_pg_pack->pages);
 176 pages_arr_err:
 177         kfree(phys_pg_pack);
 178 pages_pack_err:
 179         if (contiguous)
 180                 gen_pool_free(vm->dram_pg_pool, paddr, total_size);
 181
 182         return rc;
 183 }
 184
 185 /*
 186  * dma_map_host_va - DMA mapping of the given host virtual address.
 187  * @hdev: habanalabs device structure
 188  * @addr: the host virtual address of the memory area
 189  * @size: the size of the memory area
 190  * @p_userptr: pointer to result userptr structure
 191  *
 192  * This function does the following:
 193  * - Allocate userptr structure
 194  * - Pin the given host memory using the userptr structure
 195  * - Perform DMA mapping to have the DMA addresses of the pages
 196  */
 197 static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
 198                                 struct hl_userptr **p_userptr)
 199 {
 200         struct hl_userptr *userptr;
 201         int rc;
 202
 203         userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
 204         if (!userptr) {
 205                 rc = -ENOMEM;
 206                 goto userptr_err;
 207         }
 208
 209         rc = hl_pin_host_memory(hdev, addr, size, userptr);
 210         if (rc) {
 211                 dev_err(hdev->dev, "Failed to pin host memory\n");
 212                 goto pin_err;
 213         }
 214
 215         rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
 216                                         userptr->sgt->nents, DMA_BIDIRECTIONAL);
 217         if (rc) {
 218                 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
 219                 goto dma_map_err;
 220         }
 221
 222         userptr->dma_mapped = true;
 223         userptr->dir = DMA_BIDIRECTIONAL;
 224         userptr->vm_type = VM_TYPE_USERPTR;
 225
 226         *p_userptr = userptr;
 227
 228         return 0;
 229
 230 dma_map_err:
 231         hl_unpin_host_memory(hdev, userptr);
 232 pin_err:
 233         kfree(userptr);
 234 userptr_err:
 235
 236         return rc;
 237 }
 238
 239 /*
 240  * dma_unmap_host_va - DMA unmapping of the given host virtual address.
 241  * @hdev: habanalabs device structure
 242  * @userptr: userptr to free
 243  *
 244  * This function does the following:
 245  * - Unpins the physical pages
 246  * - Frees the userptr structure
 247  */
 248 static void dma_unmap_host_va(struct hl_device *hdev,
 249                                 struct hl_userptr *userptr)
 250 {
 251         hl_unpin_host_memory(hdev, userptr);
 252         kfree(userptr);
 253 }
 254
 255 /*
 256  * dram_pg_pool_do_release - free DRAM pages pool
 257  *
 258  * @ref                 : pointer to reference object
 259  *
 260  * This function does the following:
 261  * - Frees the idr structure of physical pages handles
 262  * - Frees the generic pool of DRAM physical pages
 263  */
 264 static void dram_pg_pool_do_release(struct kref *ref)
 265 {
 266         struct hl_vm *vm = container_of(ref, struct hl_vm,
 267                         dram_pg_pool_refcount);
 268
 269         /*
 270          * free the idr here as only here we know for sure that there are no
 271          * allocated physical pages and hence there are no handles in use
 272          */
 273         idr_destroy(&vm->phys_pg_pack_handles);
 274         gen_pool_destroy(vm->dram_pg_pool);
 275 }
 276
 277 /*
 278  * free_phys_pg_pack - free physical page pack
 279  * @hdev: habanalabs device structure
 280  * @phys_pg_pack: physical page pack to free
 281  *
 282  * This function does the following:
 283  * - For DRAM memory only, iterate over the pack and free each physical block
 284  *   structure by returning it to the general pool
 285  * - Free the hl_vm_phys_pg_pack structure
 286  */
 287 static void free_phys_pg_pack(struct hl_device *hdev,
 288                                 struct hl_vm_phys_pg_pack *phys_pg_pack)
 289 {
 290         struct hl_vm *vm = &hdev->vm;
 291         u64 i;
 292
 293         if (!phys_pg_pack->created_from_userptr) {
 294                 if (phys_pg_pack->contiguous) {
 295                         gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],
 296                                         phys_pg_pack->total_size);
 297
 298                         for (i = 0; i < phys_pg_pack->npages ; i++)
 299                                 kref_put(&vm->dram_pg_pool_refcount,
 300                                         dram_pg_pool_do_release);
 301                 } else {
 302                         for (i = 0 ; i < phys_pg_pack->npages ; i++) {
 303                                 gen_pool_free(vm->dram_pg_pool,
 304                                                 phys_pg_pack->pages[i],
 305                                                 phys_pg_pack->page_size);
 306                                 kref_put(&vm->dram_pg_pool_refcount,
 307                                         dram_pg_pool_do_release);
 308                         }
 309                 }
 310         }
 311
 312         kvfree(phys_pg_pack->pages);
 313         kfree(phys_pg_pack);
 314 }
 315
 316 /*
 317  * free_device_memory - free device memory
 318  *
 319  * @ctx                  : current context
 320  * @handle              : handle of the memory chunk to free
 321  *
 322  * This function does the following:
 323  * - Free the device memory related to the given handle
 324  */
 325 static int free_device_memory(struct hl_ctx *ctx, u32 handle)
 326 {
 327         struct hl_device *hdev = ctx->hdev;
 328         struct hl_vm *vm = &hdev->vm;
 329         struct hl_vm_phys_pg_pack *phys_pg_pack;
 330
 331         spin_lock(&vm->idr_lock);
 332         phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
 333         if (phys_pg_pack) {
 334                 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {
 335                         dev_err(hdev->dev, "handle %u is mapped, cannot free\n",
 336                                 handle);
 337                         spin_unlock(&vm->idr_lock);
 338                         return -EINVAL;
 339                 }
 340
 341                 /*
 342                  * must remove from idr before the freeing of the physical
 343                  * pages as the refcount of the pool is also the trigger of the
 344                  * idr destroy
 345                  */
 346                 idr_remove(&vm->phys_pg_pack_handles, handle);
 347                 spin_unlock(&vm->idr_lock);
 348
 349                 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);
 350                 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
 351
 352                 free_phys_pg_pack(hdev, phys_pg_pack);
 353         } else {
 354                 spin_unlock(&vm->idr_lock);
 355                 dev_err(hdev->dev,
 356                         "free device memory failed, no match for handle %u\n",
 357                         handle);
 358                 return -EINVAL;
 359         }
 360
 361         return 0;
 362 }
 363
 364 /*
 365  * clear_va_list_locked - free virtual addresses list
 366  *
 367  * @hdev                : habanalabs device structure
 368  * @va_list             : list of virtual addresses to free
 369  *
 370  * This function does the following:
 371  * - Iterate over the list and free each virtual addresses block
 372  *
 373  * This function should be called only when va_list lock is taken
 374  */
 375 static void clear_va_list_locked(struct hl_device *hdev,
 376                 struct list_head *va_list)
 377 {
 378         struct hl_vm_va_block *va_block, *tmp;
 379
 380         list_for_each_entry_safe(va_block, tmp, va_list, node) {
 381                 list_del(&va_block->node);
 382                 kfree(va_block);
 383         }
 384 }
 385
 386 /*
 387  * print_va_list_locked    - print virtual addresses list
 388  *
 389  * @hdev                : habanalabs device structure
 390  * @va_list             : list of virtual addresses to print
 391  *
 392  * This function does the following:
 393  * - Iterate over the list and print each virtual addresses block
 394  *
 395  * This function should be called only when va_list lock is taken
 396  */
 397 static void print_va_list_locked(struct hl_device *hdev,
 398                 struct list_head *va_list)
 399 {
 400 #if HL_MMU_DEBUG
 401         struct hl_vm_va_block *va_block;
 402
 403         dev_dbg(hdev->dev, "print va list:\n");
 404
 405         list_for_each_entry(va_block, va_list, node)
 406                 dev_dbg(hdev->dev,
 407                         "va block, start: 0x%llx, end: 0x%llx, size: %llu\n",
 408                         va_block->start, va_block->end, va_block->size);
 409 #endif
 410 }
 411
 412 /*
 413  * merge_va_blocks_locked - merge a virtual block if possible
 414  *
 415  * @hdev                : pointer to the habanalabs device structure
 416  * @va_list             : pointer to the virtual addresses block list
 417  * @va_block            : virtual block to merge with adjacent blocks
 418  *
 419  * This function does the following:
 420  * - Merge the given blocks with the adjacent blocks if their virtual ranges
 421  *   create a contiguous virtual range
 422  *
 423  * This Function should be called only when va_list lock is taken
 424  */
 425 static void merge_va_blocks_locked(struct hl_device *hdev,
 426                 struct list_head *va_list, struct hl_vm_va_block *va_block)
 427 {
 428         struct hl_vm_va_block *prev, *next;
 429
 430         prev = list_prev_entry(va_block, node);
 431         if (&prev->node != va_list && prev->end + 1 == va_block->start) {
 432                 prev->end = va_block->end;
 433                 prev->size = prev->end - prev->start;
 434                 list_del(&va_block->node);
 435                 kfree(va_block);
 436                 va_block = prev;
 437         }
 438
 439         next = list_next_entry(va_block, node);
 440         if (&next->node != va_list && va_block->end + 1 == next->start) {
 441                 next->start = va_block->start;
 442                 next->size = next->end - next->start;
 443                 list_del(&va_block->node);
 444                 kfree(va_block);
 445         }
 446 }
 447
 448 /*
 449  * add_va_block_locked - add a virtual block to the virtual addresses list
 450  *
 451  * @hdev                : pointer to the habanalabs device structure
 452  * @va_list             : pointer to the virtual addresses block list
 453  * @start               : start virtual address
 454  * @end                 : end virtual address
 455  *
 456  * This function does the following:
 457  * - Add the given block to the virtual blocks list and merge with other
 458  * blocks if a contiguous virtual block can be created
 459  *
 460  * This Function should be called only when va_list lock is taken
 461  */
 462 static int add_va_block_locked(struct hl_device *hdev,
 463                 struct list_head *va_list, u64 start, u64 end)
 464 {
 465         struct hl_vm_va_block *va_block, *res = NULL;
 466         u64 size = end - start;
 467
 468         print_va_list_locked(hdev, va_list);
 469
 470         list_for_each_entry(va_block, va_list, node) {
 471                 /* TODO: remove upon matureness */
 472                 if (hl_mem_area_crosses_range(start, size, va_block->start,
 473                                 va_block->end)) {
 474                         dev_err(hdev->dev,
 475                                 "block crossing ranges at start 0x%llx, end 0x%llx\n",
 476                                 va_block->start, va_block->end);
 477                         return -EINVAL;
 478                 }
 479
 480                 if (va_block->end < start)
 481                         res = va_block;
 482         }
 483
 484         va_block = kmalloc(sizeof(*va_block), GFP_KERNEL);
 485         if (!va_block)
 486                 return -ENOMEM;
 487
 488         va_block->start = start;
 489         va_block->end = end;
 490         va_block->size = size;
 491
 492         if (!res)
 493                 list_add(&va_block->node, va_list);
 494         else
 495                 list_add(&va_block->node, &res->node);
 496
 497         merge_va_blocks_locked(hdev, va_list, va_block);
 498
 499         print_va_list_locked(hdev, va_list);
 500
 501         return 0;
 502 }
 503
 504 /*
 505  * add_va_block - wrapper for add_va_block_locked
 506  *
 507  * @hdev                : pointer to the habanalabs device structure
 508  * @va_list             : pointer to the virtual addresses block list
 509  * @start               : start virtual address
 510  * @end                 : end virtual address
 511  *
 512  * This function does the following:
 513  * - Takes the list lock and calls add_va_block_locked
 514  */
 515 static inline int add_va_block(struct hl_device *hdev,
 516                 struct hl_va_range *va_range, u64 start, u64 end)
 517 {
 518         int rc;
 519
 520         mutex_lock(&va_range->lock);
 521         rc = add_va_block_locked(hdev, &va_range->list, start, end);
 522         mutex_unlock(&va_range->lock);
 523
 524         return rc;
 525 }
 526
 527 /*
 528  * get_va_block() - get a virtual block for the given size and alignment.
 529  * @hdev: pointer to the habanalabs device structure.
 530  * @va_range: pointer to the virtual addresses range.
 531  * @size: requested block size.
 532  * @hint_addr: hint for requested address by the user.
 533  * @va_block_align: required alignment of the virtual block start address.
 534  *
 535  * This function does the following:
 536  * - Iterate on the virtual block list to find a suitable virtual block for the
 537  *   given size and alignment.
 538  * - Reserve the requested block and update the list.
 539  * - Return the start address of the virtual block.
 540  */
 541 static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range,
 542                         u64 size, u64 hint_addr, u32 va_block_align)
 543 {
 544         struct hl_vm_va_block *va_block, *new_va_block = NULL;
 545         u64 valid_start, valid_size, prev_start, prev_end, align_mask,
 546                 res_valid_start = 0, res_valid_size = 0;
 547         bool add_prev = false;
 548
 549         align_mask = ~((u64)va_block_align - 1);
 550
 551         /* check if hint_addr is aligned */
 552         if (hint_addr & (va_block_align - 1))
 553                 hint_addr = 0;
 554
 555         mutex_lock(&va_range->lock);
 556
 557         print_va_list_locked(hdev, &va_range->list);
 558
 559         list_for_each_entry(va_block, &va_range->list, node) {
 560                 /* calc the first possible aligned addr */
 561                 valid_start = va_block->start;
 562
 563                 if (valid_start & (va_block_align - 1)) {
 564                         valid_start &= align_mask;
 565                         valid_start += va_block_align;
 566                         if (valid_start > va_block->end)
 567                                 continue;
 568                 }
 569
 570                 valid_size = va_block->end - valid_start;
 571
 572                 if (valid_size >= size &&
 573                         (!new_va_block || valid_size < res_valid_size)) {
 574                         new_va_block = va_block;
 575                         res_valid_start = valid_start;
 576                         res_valid_size = valid_size;
 577                 }
 578
 579                 if (hint_addr && hint_addr >= valid_start &&
 580                                 ((hint_addr + size) <= va_block->end)) {
 581                         new_va_block = va_block;
 582                         res_valid_start = hint_addr;
 583                         res_valid_size = valid_size;
 584                         break;
 585                 }
 586         }
 587
 588         if (!new_va_block) {
 589                 dev_err(hdev->dev, "no available va block for size %llu\n",
 590                                 size);
 591                 goto out;
 592         }
 593
 594         if (res_valid_start > new_va_block->start) {
 595                 prev_start = new_va_block->start;
 596                 prev_end = res_valid_start - 1;
 597
 598                 new_va_block->start = res_valid_start;
 599                 new_va_block->size = res_valid_size;
 600
 601                 add_prev = true;
 602         }
 603
 604         if (new_va_block->size > size) {
 605                 new_va_block->start += size;
 606                 new_va_block->size = new_va_block->end - new_va_block->start;
 607         } else {
 608                 list_del(&new_va_block->node);
 609                 kfree(new_va_block);
 610         }
 611
 612         if (add_prev)
 613                 add_va_block_locked(hdev, &va_range->list, prev_start,
 614                                 prev_end);
 615
 616         print_va_list_locked(hdev, &va_range->list);
 617 out:
 618         mutex_unlock(&va_range->lock);
 619
 620         return res_valid_start;
 621 }
 622
 623 /*
 624  * hl_reserve_va_block() - reserve a virtual block of a given size.
 625  * @hdev: pointer to the habanalabs device structure.
 626  * @ctx: current context
 627  * @type: virtual addresses range type.
 628  * @size: requested block size.
 629  * @alignment: required alignment in bytes of the virtual block start address,
 630  *             0 means no alignment.
 631  *
 632  * This function does the following:
 633  * - Iterate on the virtual block list to find a suitable virtual block for the
 634  *   given size and alignment.
 635  * - Reserve the requested block and update the list.
 636  * - Return the start address of the virtual block.
 637  */
 638 u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
 639                 enum hl_va_range_type type, u32 size, u32 alignment)
 640 {
 641         return get_va_block(hdev, ctx->va_range[type], size, 0,
 642                         max(alignment, ctx->va_range[type]->page_size));
 643 }
 644
 645 /**
 646  * hl_get_va_range_type() - get va_range type for the given address and size.
 647  * @address: The start address of the area we want to validate.
 648  * @size: The size in bytes of the area we want to validate.
 649  * @type: returned va_range type
 650  *
 651  * Return: true if the area is inside a valid range, false otherwise.
 652  */
 653 static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size,
 654                         enum hl_va_range_type *type)
 655 {
 656         int i;
 657
 658         for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX; i++) {
 659                 if (hl_mem_area_inside_range(address, size,
 660                                 ctx->va_range[i]->start_addr,
 661                                 ctx->va_range[i]->end_addr)) {
 662                         *type = i;
 663                         return 0;
 664                 }
 665         }
 666
 667         return -EINVAL;
 668 }
 669
 670 /*
 671  * hl_unreserve_va_block - wrapper for add_va_block for unreserving a va block
 672  *
 673  * @hdev: pointer to the habanalabs device structure
 674  * @ctx: current context
 675  * @start: start virtual address
 676  * @end: end virtual address
 677  *
 678  * This function does the following:
 679  * - Takes the list lock and calls add_va_block_locked
 680  */
 681 int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
 682                 u64 start_addr, u64 size)
 683 {
 684         enum hl_va_range_type type;
 685         int rc;
 686
 687         rc = hl_get_va_range_type(ctx, start_addr, size, &type);
 688         if (rc) {
 689                 dev_err(hdev->dev,
 690                         "cannot find va_range for va %#llx size %llu",
 691                         start_addr, size);
 692                 return rc;
 693         }
 694
 695         rc = add_va_block(hdev, ctx->va_range[type], start_addr,
 696                                                 start_addr + size - 1);
 697         if (rc)
 698                 dev_warn(hdev->dev,
 699                         "add va block failed for vaddr: 0x%llx\n", start_addr);
 700
 701         return rc;
 702 }
 703
 704 /*
 705  * get_sg_info - get number of pages and the DMA address from SG list
 706  *
 707  * @sg                 : the SG list
 708  * @dma_addr           : pointer to DMA address to return
 709  *
 710  * Calculate the number of consecutive pages described by the SG list. Take the
 711  * offset of the address in the first page, add to it the length and round it up
 712  * to the number of needed pages.
 713  */
 714 static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
 715 {
 716         *dma_addr = sg_dma_address(sg);
 717
 718         return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
 719                         (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 720 }
 721
 722 /*
 723  * init_phys_pg_pack_from_userptr - initialize physical page pack from host
 724  *                                  memory
 725  * @ctx: current context
 726  * @userptr: userptr to initialize from
 727  * @pphys_pg_pack: result pointer
 728  *
 729  * This function does the following:
 730  * - Pin the physical pages related to the given virtual block
 731  * - Create a physical page pack from the physical pages related to the given
 732  *   virtual block
 733  */
 734 static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
 735                                 struct hl_userptr *userptr,
 736                                 struct hl_vm_phys_pg_pack **pphys_pg_pack)
 737 {
 738         struct hl_vm_phys_pg_pack *phys_pg_pack;
 739         struct scatterlist *sg;
 740         dma_addr_t dma_addr;
 741         u64 page_mask, total_npages;
 742         u32 npages, page_size = PAGE_SIZE,
 743                 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
 744         bool first = true, is_huge_page_opt = true;
 745         int rc, i, j;
 746         u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
 747
 748         phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
 749         if (!phys_pg_pack)
 750                 return -ENOMEM;
 751
 752         phys_pg_pack->vm_type = userptr->vm_type;
 753         phys_pg_pack->created_from_userptr = true;
 754         phys_pg_pack->asid = ctx->asid;
 755         atomic_set(&phys_pg_pack->mapping_cnt, 1);
 756
 757         /* Only if all dma_addrs are aligned to 2MB and their
 758          * sizes is at least 2MB, we can use huge page mapping.
 759          * We limit the 2MB optimization to this condition,
 760          * since later on we acquire the related VA range as one
 761          * consecutive block.
 762          */
 763         total_npages = 0;
 764         for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
 765                 npages = get_sg_info(sg, &dma_addr);
 766
 767                 total_npages += npages;
 768
 769                 if ((npages % pgs_in_huge_page) ||
 770                                         (dma_addr & (huge_page_size - 1)))
 771                         is_huge_page_opt = false;
 772         }
 773
 774         if (is_huge_page_opt) {
 775                 page_size = huge_page_size;
 776                 do_div(total_npages, pgs_in_huge_page);
 777         }
 778
 779         page_mask = ~(((u64) page_size) - 1);
 780
 781         phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),
 782                                                 GFP_KERNEL);
 783         if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
 784                 rc = -ENOMEM;
 785                 goto page_pack_arr_mem_err;
 786         }
 787
 788         phys_pg_pack->npages = total_npages;
 789         phys_pg_pack->page_size = page_size;
 790         phys_pg_pack->total_size = total_npages * page_size;
 791
 792         j = 0;
 793         for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
 794                 npages = get_sg_info(sg, &dma_addr);
 795
 796                 /* align down to physical page size and save the offset */
 797                 if (first) {
 798                         first = false;
 799                         phys_pg_pack->offset = dma_addr & (page_size - 1);
 800                         dma_addr &= page_mask;
 801                 }
 802
 803                 while (npages) {
 804                         phys_pg_pack->pages[j++] = dma_addr;
 805                         dma_addr += page_size;
 806
 807                         if (is_huge_page_opt)
 808                                 npages -= pgs_in_huge_page;
 809                         else
 810                                 npages--;
 811                 }
 812         }
 813
 814         *pphys_pg_pack = phys_pg_pack;
 815
 816         return 0;
 817
 818 page_pack_arr_mem_err:
 819         kfree(phys_pg_pack);
 820
 821         return rc;
 822 }
 823
 824 /*
 825  * map_phys_pg_pack - maps the physical page pack.
 826  * @ctx: current context
 827  * @vaddr: start address of the virtual area to map from
 828  * @phys_pg_pack: the pack of physical pages to map to
 829  *
 830  * This function does the following:
 831  * - Maps each chunk of virtual memory to matching physical chunk
 832  * - Stores number of successful mappings in the given argument
 833  * - Returns 0 on success, error code otherwise
 834  */
 835 static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
 836                                 struct hl_vm_phys_pg_pack *phys_pg_pack)
 837 {
 838         struct hl_device *hdev = ctx->hdev;
 839         u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;
 840         u32 page_size = phys_pg_pack->page_size;
 841         int rc = 0;
 842
 843         for (i = 0 ; i < phys_pg_pack->npages ; i++) {
 844                 paddr = phys_pg_pack->pages[i];
 845
 846                 rc = hl_mmu_map_page(ctx, next_vaddr, paddr, page_size,
 847                                 (i + 1) == phys_pg_pack->npages);
 848                 if (rc) {
 849                         dev_err(hdev->dev,
 850                                 "map failed for handle %u, npages: %llu, mapped: %llu",
 851                                 phys_pg_pack->handle, phys_pg_pack->npages,
 852                                 mapped_pg_cnt);
 853                         goto err;
 854                 }
 855
 856                 mapped_pg_cnt++;
 857                 next_vaddr += page_size;
 858         }
 859
 860         return 0;
 861
 862 err:
 863         next_vaddr = vaddr;
 864         for (i = 0 ; i < mapped_pg_cnt ; i++) {
 865                 if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
 866                                         (i + 1) == mapped_pg_cnt))
 867                         dev_warn_ratelimited(hdev->dev,
 868                                 "failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n",
 869                                         phys_pg_pack->handle, next_vaddr,
 870                                         phys_pg_pack->pages[i], page_size);
 871
 872                 next_vaddr += page_size;
 873         }
 874
 875         return rc;
 876 }
 877
 878 /*
 879  * unmap_phys_pg_pack - unmaps the physical page pack
 880  * @ctx: current context
 881  * @vaddr: start address of the virtual area to unmap
 882  * @phys_pg_pack: the pack of physical pages to unmap
 883  */
 884 static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
 885                                 struct hl_vm_phys_pg_pack *phys_pg_pack)
 886 {
 887         struct hl_device *hdev = ctx->hdev;
 888         u64 next_vaddr, i;
 889         bool is_host_addr;
 890         u32 page_size;
 891
 892         is_host_addr = !hl_is_dram_va(hdev, vaddr);
 893         page_size = phys_pg_pack->page_size;
 894         next_vaddr = vaddr;
 895
 896         for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {
 897                 if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
 898                                        (i + 1) == phys_pg_pack->npages))
 899                         dev_warn_ratelimited(hdev->dev,
 900                         "unmap failed for vaddr: 0x%llx\n", next_vaddr);
 901
 902                 /*
 903                  * unmapping on Palladium can be really long, so avoid a CPU
 904                  * soft lockup bug by sleeping a little between unmapping pages
 905                  *
 906                  * In addition, when unmapping host memory we pass through
 907                  * the Linux kernel to unpin the pages and that takes a long
 908                  * time. Therefore, sleep every 32K pages to avoid soft lockup
 909                  */
 910                 if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0))
 911                         usleep_range(50, 200);
 912         }
 913 }
 914
 915 static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args,
 916                                 u64 *paddr)
 917 {
 918         struct hl_device *hdev = ctx->hdev;
 919         struct hl_vm *vm = &hdev->vm;
 920         struct hl_vm_phys_pg_pack *phys_pg_pack;
 921         u32 handle;
 922
 923         handle = lower_32_bits(args->map_device.handle);
 924         spin_lock(&vm->idr_lock);
 925         phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
 926         if (!phys_pg_pack) {
 927                 spin_unlock(&vm->idr_lock);
 928                 dev_err(hdev->dev, "no match for handle %u\n", handle);
 929                 return -EINVAL;
 930         }
 931
 932         *paddr = phys_pg_pack->pages[0];
 933
 934         spin_unlock(&vm->idr_lock);
 935
 936         return 0;
 937 }
 938
 939 /*
 940  * map_device_va - map the given memory
 941  *
 942  * @ctx          : current context
 943  * @args         : host parameters with handle/host virtual address
 944  * @device_addr  : pointer to result device virtual address
 945  *
 946  * This function does the following:
 947  * - If given a physical device memory handle, map to a device virtual block
 948  *   and return the start address of this block
 949  * - If given a host virtual address and size, find the related physical pages,
 950  *   map a device virtual block to this pages and return the start address of
 951  *   this block
 952  */
 953 static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
 954                 u64 *device_addr)
 955 {
 956         struct hl_device *hdev = ctx->hdev;
 957         struct hl_vm *vm = &hdev->vm;
 958         struct hl_vm_phys_pg_pack *phys_pg_pack;
 959         struct hl_userptr *userptr = NULL;
 960         struct hl_vm_hash_node *hnode;
 961         struct hl_va_range *va_range;
 962         enum vm_type_t *vm_type;
 963         u64 ret_vaddr, hint_addr;
 964         u32 handle = 0, va_block_align;
 965         int rc;
 966         bool is_userptr = args->flags & HL_MEM_USERPTR;
 967
 968         /* Assume failure */
 969         *device_addr = 0;
 970
 971         if (is_userptr) {
 972                 u64 addr = args->map_host.host_virt_addr,
 973                         size = args->map_host.mem_size;
 974                 u32 page_size = hdev->asic_prop.pmmu.page_size,
 975                         huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
 976
 977                 rc = dma_map_host_va(hdev, addr, size, &userptr);
 978                 if (rc) {
 979                         dev_err(hdev->dev, "failed to get userptr from va\n");
 980                         return rc;
 981                 }
 982
 983                 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
 984                                 &phys_pg_pack);
 985                 if (rc) {
 986                         dev_err(hdev->dev,
 987                                 "unable to init page pack for vaddr 0x%llx\n",
 988                                 addr);
 989                         goto init_page_pack_err;
 990                 }
 991
 992                 vm_type = (enum vm_type_t *) userptr;
 993                 hint_addr = args->map_host.hint_addr;
 994                 handle = phys_pg_pack->handle;
 995
 996                 /* get required alignment */
 997                 if (phys_pg_pack->page_size == page_size) {
 998                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
 999
1000                         /*
1001                          * huge page alignment may be needed in case of regular
1002                          * page mapping, depending on the host VA alignment
1003                          */
1004                         if (addr & (huge_page_size - 1))
1005                                 va_block_align = page_size;
1006                         else
1007                                 va_block_align = huge_page_size;
1008                 } else {
1009                         /*
1010                          * huge page alignment is needed in case of huge page
1011                          * mapping
1012                          */
1013                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1014                         va_block_align = huge_page_size;
1015                 }
1016         } else {
1017                 handle = lower_32_bits(args->map_device.handle);
1018
1019                 spin_lock(&vm->idr_lock);
1020                 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
1021                 if (!phys_pg_pack) {
1022                         spin_unlock(&vm->idr_lock);
1023                         dev_err(hdev->dev,
1024                                 "no match for handle %u\n", handle);
1025                         return -EINVAL;
1026                 }
1027
1028                 /* increment now to avoid freeing device memory while mapping */
1029                 atomic_inc(&phys_pg_pack->mapping_cnt);
1030
1031                 spin_unlock(&vm->idr_lock);
1032
1033                 vm_type = (enum vm_type_t *) phys_pg_pack;
1034
1035                 hint_addr = args->map_device.hint_addr;
1036
1037                 /* DRAM VA alignment is the same as the DRAM page size */
1038                 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1039                 va_block_align = hdev->asic_prop.dmmu.page_size;
1040         }
1041
1042         /*
1043          * relevant for mapping device physical memory only, as host memory is
1044          * implicitly shared
1045          */
1046         if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&
1047                         phys_pg_pack->asid != ctx->asid) {
1048                 dev_err(hdev->dev,
1049                         "Failed to map memory, handle %u is not shared\n",
1050                         handle);
1051                 rc = -EPERM;
1052                 goto shared_err;
1053         }
1054
1055         hnode = kzalloc(sizeof(*hnode), GFP_KERNEL);
1056         if (!hnode) {
1057                 rc = -ENOMEM;
1058                 goto hnode_err;
1059         }
1060
1061         ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
1062                                         hint_addr, va_block_align);
1063         if (!ret_vaddr) {
1064                 dev_err(hdev->dev, "no available va block for handle %u\n",
1065                                 handle);
1066                 rc = -ENOMEM;
1067                 goto va_block_err;
1068         }
1069
1070         mutex_lock(&ctx->mmu_lock);
1071
1072         rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
1073         if (rc) {
1074                 mutex_unlock(&ctx->mmu_lock);
1075                 dev_err(hdev->dev, "mapping page pack failed for handle %u\n",
1076                                 handle);
1077                 goto map_err;
1078         }
1079
1080         rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
1081
1082         mutex_unlock(&ctx->mmu_lock);
1083
1084         if (rc) {
1085                 dev_err(hdev->dev,
1086                         "mapping handle %u failed due to MMU cache invalidation\n",
1087                         handle);
1088                 goto map_err;
1089         }
1090
1091         ret_vaddr += phys_pg_pack->offset;
1092
1093         hnode->ptr = vm_type;
1094         hnode->vaddr = ret_vaddr;
1095
1096         mutex_lock(&ctx->mem_hash_lock);
1097         hash_add(ctx->mem_hash, &hnode->node, ret_vaddr);
1098         mutex_unlock(&ctx->mem_hash_lock);
1099
1100         *device_addr = ret_vaddr;
1101
1102         if (is_userptr)
1103                 free_phys_pg_pack(hdev, phys_pg_pack);
1104
1105         return 0;
1106
1107 map_err:
1108         if (add_va_block(hdev, va_range, ret_vaddr,
1109                                 ret_vaddr + phys_pg_pack->total_size - 1))
1110                 dev_warn(hdev->dev,
1111                         "release va block failed for handle 0x%x, vaddr: 0x%llx\n",
1112                                 handle, ret_vaddr);
1113
1114 va_block_err:
1115         kfree(hnode);
1116 hnode_err:
1117 shared_err:
1118         atomic_dec(&phys_pg_pack->mapping_cnt);
1119         if (is_userptr)
1120                 free_phys_pg_pack(hdev, phys_pg_pack);
1121 init_page_pack_err:
1122         if (is_userptr)
1123                 dma_unmap_host_va(hdev, userptr);
1124
1125         return rc;
1126 }
1127
1128 /*
1129  * unmap_device_va      - unmap the given device virtual address
1130  *
1131  * @ctx                 : current context
1132  * @vaddr               : device virtual address to unmap
1133  * @ctx_free            : true if in context free flow, false otherwise.
1134  *
1135  * This function does the following:
1136  * - Unmap the physical pages related to the given virtual address
1137  * - return the device virtual block to the virtual block list
1138  */
1139 static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
1140 {
1141         struct hl_device *hdev = ctx->hdev;
1142         struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
1143         struct hl_vm_hash_node *hnode = NULL;
1144         struct hl_userptr *userptr = NULL;
1145         struct hl_va_range *va_range;
1146         enum vm_type_t *vm_type;
1147         bool is_userptr;
1148         int rc = 0;
1149
1150         /* protect from double entrance */
1151         mutex_lock(&ctx->mem_hash_lock);
1152         hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
1153                 if (vaddr == hnode->vaddr)
1154                         break;
1155
1156         if (!hnode) {
1157                 mutex_unlock(&ctx->mem_hash_lock);
1158                 dev_err(hdev->dev,
1159                         "unmap failed, no mem hnode for vaddr 0x%llx\n",
1160                         vaddr);
1161                 return -EINVAL;
1162         }
1163
1164         hash_del(&hnode->node);
1165         mutex_unlock(&ctx->mem_hash_lock);
1166
1167         vm_type = hnode->ptr;
1168
1169         if (*vm_type == VM_TYPE_USERPTR) {
1170                 is_userptr = true;
1171                 userptr = hnode->ptr;
1172                 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1173                                                         &phys_pg_pack);
1174                 if (rc) {
1175                         dev_err(hdev->dev,
1176                                 "unable to init page pack for vaddr 0x%llx\n",
1177                                 vaddr);
1178                         goto vm_type_err;
1179                 }
1180
1181                 if (phys_pg_pack->page_size ==
1182                                         hdev->asic_prop.pmmu.page_size)
1183                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1184                 else
1185                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1186         } else if (*vm_type == VM_TYPE_PHYS_PACK) {
1187                 is_userptr = false;
1188                 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1189                 phys_pg_pack = hnode->ptr;
1190         } else {
1191                 dev_warn(hdev->dev,
1192                         "unmap failed, unknown vm desc for vaddr 0x%llx\n",
1193                                 vaddr);
1194                 rc = -EFAULT;
1195                 goto vm_type_err;
1196         }
1197
1198         if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {
1199                 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr);
1200                 rc = -EINVAL;
1201                 goto mapping_cnt_err;
1202         }
1203
1204         vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);
1205
1206         mutex_lock(&ctx->mmu_lock);
1207
1208         unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);
1209
1210         /*
1211          * During context free this function is called in a loop to clean all
1212          * the context mappings. Hence the cache invalidation can be called once
1213          * at the loop end rather than for each iteration
1214          */
1215         if (!ctx_free)
1216                 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
1217                                                                 *vm_type);
1218
1219         mutex_unlock(&ctx->mmu_lock);
1220
1221         /*
1222          * If the context is closing we don't need to check for the MMU cache
1223          * invalidation return code and update the VA free list as in this flow
1224          * we invalidate the MMU cache outside of this unmap function and the VA
1225          * free list will be freed anyway.
1226          */
1227         if (!ctx_free) {
1228                 int tmp_rc;
1229
1230                 if (rc)
1231                         dev_err(hdev->dev,
1232                                 "unmapping vaddr 0x%llx failed due to MMU cache invalidation\n",
1233                                 vaddr);
1234
1235                 tmp_rc = add_va_block(hdev, va_range, vaddr,
1236                                         vaddr + phys_pg_pack->total_size - 1);
1237                 if (tmp_rc) {
1238                         dev_warn(hdev->dev,
1239                                         "add va block failed for vaddr: 0x%llx\n",
1240                                         vaddr);
1241                         if (!rc)
1242                                 rc = tmp_rc;
1243                 }
1244         }
1245
1246         atomic_dec(&phys_pg_pack->mapping_cnt);
1247         kfree(hnode);
1248
1249         if (is_userptr) {
1250                 free_phys_pg_pack(hdev, phys_pg_pack);
1251                 dma_unmap_host_va(hdev, userptr);
1252         }
1253
1254         return rc;
1255
1256 mapping_cnt_err:
1257         if (is_userptr)
1258                 free_phys_pg_pack(hdev, phys_pg_pack);
1259 vm_type_err:
1260         mutex_lock(&ctx->mem_hash_lock);
1261         hash_add(ctx->mem_hash, &hnode->node, vaddr);
1262         mutex_unlock(&ctx->mem_hash_lock);
1263
1264         return rc;
1265 }
1266
1267 static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args)
1268 {
1269         struct hl_device *hdev = hpriv->hdev;
1270         struct hl_ctx *ctx = hpriv->ctx;
1271         u64 device_addr = 0;
1272         u32 handle = 0;
1273         int rc;
1274
1275         switch (args->in.op) {
1276         case HL_MEM_OP_ALLOC:
1277                 if (args->in.alloc.mem_size == 0) {
1278                         dev_err(hdev->dev,
1279                                 "alloc size must be larger than 0\n");
1280                         rc = -EINVAL;
1281                         goto out;
1282                 }
1283
1284                 /* Force contiguous as there are no real MMU
1285                  * translations to overcome physical memory gaps
1286                  */
1287                 args->in.flags |= HL_MEM_CONTIGUOUS;
1288                 rc = alloc_device_memory(ctx, &args->in, &handle);
1289
1290                 memset(args, 0, sizeof(*args));
1291                 args->out.handle = (__u64) handle;
1292                 break;
1293
1294         case HL_MEM_OP_FREE:
1295                 rc = free_device_memory(ctx, args->in.free.handle);
1296                 break;
1297
1298         case HL_MEM_OP_MAP:
1299                 if (args->in.flags & HL_MEM_USERPTR) {
1300                         device_addr = args->in.map_host.host_virt_addr;
1301                         rc = 0;
1302                 } else {
1303                         rc = get_paddr_from_handle(ctx, &args->in,
1304                                         &device_addr);
1305                 }
1306
1307                 memset(args, 0, sizeof(*args));
1308                 args->out.device_virt_addr = device_addr;
1309                 break;
1310
1311         case HL_MEM_OP_UNMAP:
1312                 rc = 0;
1313                 break;
1314
1315         default:
1316                 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1317                 rc = -ENOTTY;
1318                 break;
1319         }
1320
1321 out:
1322         return rc;
1323 }
1324
1325 int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
1326 {
1327         enum hl_device_status status;
1328         union hl_mem_args *args = data;
1329         struct hl_device *hdev = hpriv->hdev;
1330         struct hl_ctx *ctx = hpriv->ctx;
1331         u64 device_addr = 0;
1332         u32 handle = 0;
1333         int rc;
1334
1335         if (!hl_device_operational(hdev, &status)) {
1336                 dev_warn_ratelimited(hdev->dev,
1337                         "Device is %s. Can't execute MEMORY IOCTL\n",
1338                         hdev->status[status]);
1339                 return -EBUSY;
1340         }
1341
1342         if (!hdev->mmu_enable)
1343                 return mem_ioctl_no_mmu(hpriv, args);
1344
1345         switch (args->in.op) {
1346         case HL_MEM_OP_ALLOC:
1347                 if (args->in.alloc.mem_size == 0) {
1348                         dev_err(hdev->dev,
1349                                 "alloc size must be larger than 0\n");
1350                         rc = -EINVAL;
1351                         goto out;
1352                 }
1353
1354                 /* If DRAM does not support virtual memory the driver won't
1355                  * handle the allocation/freeing of that memory. However, for
1356                  * system administration/monitoring purposes, the driver will
1357                  * keep track of the amount of DRAM memory that is allocated
1358                  * and freed by the user. Because this code totally relies on
1359                  * the user's input, the driver can't ensure the validity
1360                  * of this accounting.
1361                  */
1362                 if (!hdev->asic_prop.dram_supports_virtual_memory) {
1363                         atomic64_add(args->in.alloc.mem_size,
1364                                         &ctx->dram_phys_mem);
1365                         atomic64_add(args->in.alloc.mem_size,
1366                                         &hdev->dram_used_mem);
1367
1368                         dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
1369                         rc = 0;
1370
1371                         memset(args, 0, sizeof(*args));
1372                         args->out.handle = 0;
1373                         goto out;
1374                 }
1375
1376                 rc = alloc_device_memory(ctx, &args->in, &handle);
1377
1378                 memset(args, 0, sizeof(*args));
1379                 args->out.handle = (__u64) handle;
1380                 break;
1381
1382         case HL_MEM_OP_FREE:
1383                 /* If DRAM does not support virtual memory the driver won't
1384                  * handle the allocation/freeing of that memory. However, for
1385                  * system administration/monitoring purposes, the driver will
1386                  * keep track of the amount of DRAM memory that is allocated
1387                  * and freed by the user. Because this code totally relies on
1388                  * the user's input, the driver can't ensure the validity
1389                  * of this accounting.
1390                  */
1391                 if (!hdev->asic_prop.dram_supports_virtual_memory) {
1392                         atomic64_sub(args->in.alloc.mem_size,
1393                                         &ctx->dram_phys_mem);
1394                         atomic64_sub(args->in.alloc.mem_size,
1395                                         &hdev->dram_used_mem);
1396
1397                         dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
1398                         rc = 0;
1399
1400                         goto out;
1401                 }
1402
1403                 rc = free_device_memory(ctx, args->in.free.handle);
1404                 break;
1405
1406         case HL_MEM_OP_MAP:
1407                 rc = map_device_va(ctx, &args->in, &device_addr);
1408
1409                 memset(args, 0, sizeof(*args));
1410                 args->out.device_virt_addr = device_addr;
1411                 break;
1412
1413         case HL_MEM_OP_UNMAP:
1414                 rc = unmap_device_va(ctx, args->in.unmap.device_virt_addr,
1415                                         false);
1416                 break;
1417
1418         default:
1419                 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1420                 rc = -ENOTTY;
1421                 break;
1422         }
1423
1424 out:
1425         return rc;
1426 }
1427
1428 static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
1429                                 u32 npages, u64 start, u32 offset,
1430                                 struct hl_userptr *userptr)
1431 {
1432         int rc;
1433
1434         if (!access_ok((void __user *) (uintptr_t) addr, size)) {
1435                 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr);
1436                 return -EFAULT;
1437         }
1438
1439         userptr->vec = frame_vector_create(npages);
1440         if (!userptr->vec) {
1441                 dev_err(hdev->dev, "Failed to create frame vector\n");
1442                 return -ENOMEM;
1443         }
1444
1445         rc = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
1446                                 userptr->vec);
1447
1448         if (rc != npages) {
1449                 dev_err(hdev->dev,
1450                         "Failed to map host memory, user ptr probably wrong\n");
1451                 if (rc < 0)
1452                         goto destroy_framevec;
1453                 rc = -EFAULT;
1454                 goto put_framevec;
1455         }
1456
1457         if (frame_vector_to_pages(userptr->vec) < 0) {
1458                 dev_err(hdev->dev,
1459                         "Failed to translate frame vector to pages\n");
1460                 rc = -EFAULT;
1461                 goto put_framevec;
1462         }
1463
1464         rc = sg_alloc_table_from_pages(userptr->sgt,
1465                                         frame_vector_pages(userptr->vec),
1466                                         npages, offset, size, GFP_ATOMIC);
1467         if (rc < 0) {
1468                 dev_err(hdev->dev, "failed to create SG table from pages\n");
1469                 goto put_framevec;
1470         }
1471
1472         return 0;
1473
1474 put_framevec:
1475         put_vaddr_frames(userptr->vec);
1476 destroy_framevec:
1477         frame_vector_destroy(userptr->vec);
1478         return rc;
1479 }
1480
1481 /*
1482  * hl_pin_host_memory - pins a chunk of host memory.
1483  * @hdev: pointer to the habanalabs device structure
1484  * @addr: the host virtual address of the memory area
1485  * @size: the size of the memory area
1486  * @userptr: pointer to hl_userptr structure
1487  *
1488  * This function does the following:
1489  * - Pins the physical pages
1490  * - Create an SG list from those pages
1491  */
1492 int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1493                                         struct hl_userptr *userptr)
1494 {
1495         u64 start, end;
1496         u32 npages, offset;
1497         int rc;
1498
1499         if (!size) {
1500                 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size);
1501                 return -EINVAL;
1502         }
1503
1504         /*
1505          * If the combination of the address and size requested for this memory
1506          * region causes an integer overflow, return error.
1507          */
1508         if (((addr + size) < addr) ||
1509                         PAGE_ALIGN(addr + size) < (addr + size)) {
1510                 dev_err(hdev->dev,
1511                         "user pointer 0x%llx + %llu causes integer overflow\n",
1512                         addr, size);
1513                 return -EINVAL;
1514         }
1515
1516         /*
1517          * This function can be called also from data path, hence use atomic
1518          * always as it is not a big allocation.
1519          */
1520         userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_ATOMIC);
1521         if (!userptr->sgt)
1522                 return -ENOMEM;
1523
1524         start = addr & PAGE_MASK;
1525         offset = addr & ~PAGE_MASK;
1526         end = PAGE_ALIGN(addr + size);
1527         npages = (end - start) >> PAGE_SHIFT;
1528
1529         userptr->size = size;
1530         userptr->addr = addr;
1531         userptr->dma_mapped = false;
1532         INIT_LIST_HEAD(&userptr->job_node);
1533
1534         rc = get_user_memory(hdev, addr, size, npages, start, offset,
1535                                 userptr);
1536         if (rc) {
1537                 dev_err(hdev->dev,
1538                         "failed to get user memory for address 0x%llx\n",
1539                         addr);
1540                 goto free_sgt;
1541         }
1542
1543         hl_debugfs_add_userptr(hdev, userptr);
1544
1545         return 0;
1546
1547 free_sgt:
1548         kfree(userptr->sgt);
1549         return rc;
1550 }
1551
1552 /*
1553  * hl_unpin_host_memory - unpins a chunk of host memory.
1554  * @hdev: pointer to the habanalabs device structure
1555  * @userptr: pointer to hl_userptr structure
1556  *
1557  * This function does the following:
1558  * - Unpins the physical pages related to the host memory
1559  * - Free the SG list
1560  */
1561 void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
1562 {
1563         struct page **pages;
1564
1565         hl_debugfs_remove_userptr(hdev, userptr);
1566
1567         if (userptr->dma_mapped)
1568                 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl,
1569                                                         userptr->sgt->nents,
1570                                                         userptr->dir);
1571
1572         pages = frame_vector_pages(userptr->vec);
1573         if (!IS_ERR(pages)) {
1574                 int i;
1575
1576                 for (i = 0; i < frame_vector_count(userptr->vec); i++)
1577                         set_page_dirty_lock(pages[i]);
1578         }
1579         put_vaddr_frames(userptr->vec);
1580         frame_vector_destroy(userptr->vec);
1581
1582         list_del(&userptr->job_node);
1583
1584         sg_free_table(userptr->sgt);
1585         kfree(userptr->sgt);
1586 }
1587
1588 /*
1589  * hl_userptr_delete_list - clear userptr list
1590  *
1591  * @hdev                : pointer to the habanalabs device structure
1592  * @userptr_list        : pointer to the list to clear
1593  *
1594  * This function does the following:
1595  * - Iterates over the list and unpins the host memory and frees the userptr
1596  *   structure.
1597  */
1598 void hl_userptr_delete_list(struct hl_device *hdev,
1599                                 struct list_head *userptr_list)
1600 {
1601         struct hl_userptr *userptr, *tmp;
1602
1603         list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) {
1604                 hl_unpin_host_memory(hdev, userptr);
1605                 kfree(userptr);
1606         }
1607
1608         INIT_LIST_HEAD(userptr_list);
1609 }
1610
1611 /*
1612  * hl_userptr_is_pinned - returns whether the given userptr is pinned
1613  *
1614  * @hdev                : pointer to the habanalabs device structure
1615  * @userptr_list        : pointer to the list to clear
1616  * @userptr             : pointer to userptr to check
1617  *
1618  * This function does the following:
1619  * - Iterates over the list and checks if the given userptr is in it, means is
1620  *   pinned. If so, returns true, otherwise returns false.
1621  */
1622 bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
1623                                 u32 size, struct list_head *userptr_list,
1624                                 struct hl_userptr **userptr)
1625 {
1626         list_for_each_entry((*userptr), userptr_list, job_node) {
1627                 if ((addr == (*userptr)->addr) && (size == (*userptr)->size))
1628                         return true;
1629         }
1630
1631         return false;
1632 }
1633
1634 /*
1635  * va_range_init - initialize virtual addresses range
1636  * @hdev: pointer to the habanalabs device structure
1637  * @va_range: pointer to the range to initialize
1638  * @start: range start address
1639  * @end: range end address
1640  *
1641  * This function does the following:
1642  * - Initializes the virtual addresses list of the given range with the given
1643  *   addresses.
1644  */
1645 static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
1646                                 u64 start, u64 end, u32 page_size)
1647 {
1648         int rc;
1649
1650         INIT_LIST_HEAD(&va_range->list);
1651
1652         /* PAGE_SIZE alignment */
1653
1654         if (start & (PAGE_SIZE - 1)) {
1655                 start &= PAGE_MASK;
1656                 start += PAGE_SIZE;
1657         }
1658
1659         if (end & (PAGE_SIZE - 1))
1660                 end &= PAGE_MASK;
1661
1662         if (start >= end) {
1663                 dev_err(hdev->dev, "too small vm range for va list\n");
1664                 return -EFAULT;
1665         }
1666
1667         rc = add_va_block(hdev, va_range, start, end);
1668
1669         if (rc) {
1670                 dev_err(hdev->dev, "Failed to init host va list\n");
1671                 return rc;
1672         }
1673
1674         va_range->start_addr = start;
1675         va_range->end_addr = end;
1676         va_range->page_size = page_size;
1677
1678         return 0;
1679 }
1680
1681 /*
1682  * va_range_fini() - clear a virtual addresses range
1683  * @hdev: pointer to the habanalabs structure
1684  * va_range: pointer to virtual addresses range
1685  *
1686  * This function does the following:
1687  * - Frees the virtual addresses block list and its lock
1688  */
1689 static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range)
1690 {
1691         mutex_lock(&va_range->lock);
1692         clear_va_list_locked(hdev, &va_range->list);
1693         mutex_unlock(&va_range->lock);
1694
1695         mutex_destroy(&va_range->lock);
1696         kfree(va_range);
1697 }
1698
1699 /*
1700  * vm_ctx_init_with_ranges() - initialize virtual memory for context
1701  * @ctx: pointer to the habanalabs context structure
1702  * @host_range_start: host virtual addresses range start.
1703  * @host_range_end: host virtual addresses range end.
1704  * @host_huge_range_start: host virtual addresses range start for memory
1705  *                          allocated with huge pages.
1706  * @host_huge_range_end: host virtual addresses range end for memory allocated
1707  *                        with huge pages.
1708  * @dram_range_start: dram virtual addresses range start.
1709  * @dram_range_end: dram virtual addresses range end.
1710  *
1711  * This function initializes the following:
1712  * - MMU for context
1713  * - Virtual address to area descriptor hashtable
1714  * - Virtual block list of available virtual memory
1715  */
1716 static int vm_ctx_init_with_ranges(struct hl_ctx *ctx,
1717                                         u64 host_range_start,
1718                                         u64 host_range_end,
1719                                         u32 host_page_size,
1720                                         u64 host_huge_range_start,
1721                                         u64 host_huge_range_end,
1722                                         u32 host_huge_page_size,
1723                                         u64 dram_range_start,
1724                                         u64 dram_range_end,
1725                                         u32 dram_page_size)
1726 {
1727         struct hl_device *hdev = ctx->hdev;
1728         int i, rc;
1729
1730         for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) {
1731                 ctx->va_range[i] =
1732                         kzalloc(sizeof(struct hl_va_range), GFP_KERNEL);
1733                 if (!ctx->va_range[i]) {
1734                         rc = -ENOMEM;
1735                         goto free_va_range;
1736                 }
1737         }
1738
1739         rc = hl_mmu_ctx_init(ctx);
1740         if (rc) {
1741                 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
1742                 goto free_va_range;
1743         }
1744
1745         mutex_init(&ctx->mem_hash_lock);
1746         hash_init(ctx->mem_hash);
1747
1748         mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1749
1750         rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST],
1751                         host_range_start, host_range_end, host_page_size);
1752         if (rc) {
1753                 dev_err(hdev->dev, "failed to init host vm range\n");
1754                 goto mmu_ctx_fini;
1755         }
1756
1757         if (hdev->pmmu_huge_range) {
1758                 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1759
1760                 rc = va_range_init(hdev,
1761                         ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE],
1762                         host_huge_range_start, host_huge_range_end,
1763                         host_huge_page_size);
1764                 if (rc) {
1765                         dev_err(hdev->dev,
1766                                 "failed to init host huge vm range\n");
1767                         goto clear_host_va_range;
1768                 }
1769         } else {
1770                 kfree(ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
1771                 ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] =
1772                                 ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1773         }
1774
1775         mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
1776
1777         rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM],
1778                         dram_range_start, dram_range_end, dram_page_size);
1779         if (rc) {
1780                 dev_err(hdev->dev, "failed to init dram vm range\n");
1781                 goto clear_host_huge_va_range;
1782         }
1783
1784         hl_debugfs_add_ctx_mem_hash(hdev, ctx);
1785
1786         return 0;
1787
1788 clear_host_huge_va_range:
1789         mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
1790
1791         if (hdev->pmmu_huge_range) {
1792                 mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1793                 clear_va_list_locked(hdev,
1794                         &ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list);
1795                 mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1796         }
1797 clear_host_va_range:
1798         if (hdev->pmmu_huge_range)
1799                 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1800         mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1801         clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list);
1802         mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1803 mmu_ctx_fini:
1804         mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1805         mutex_destroy(&ctx->mem_hash_lock);
1806         hl_mmu_ctx_fini(ctx);
1807 free_va_range:
1808         for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++)
1809                 kfree(ctx->va_range[i]);
1810
1811         return rc;
1812 }
1813
1814 int hl_vm_ctx_init(struct hl_ctx *ctx)
1815 {
1816         struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
1817         u64 host_range_start, host_range_end, host_huge_range_start,
1818                 host_huge_range_end, dram_range_start, dram_range_end;
1819         u32 host_page_size, host_huge_page_size, dram_page_size;
1820
1821         atomic64_set(&ctx->dram_phys_mem, 0);
1822
1823         /*
1824          * - If MMU is enabled, init the ranges as usual.
1825          * - If MMU is disabled, in case of host mapping, the returned address
1826          *   is the given one.
1827          *   In case of DRAM mapping, the returned address is the physical
1828          *   address of the memory related to the given handle.
1829          */
1830         if (!ctx->hdev->mmu_enable)
1831                 return 0;
1832
1833         dram_range_start = prop->dmmu.start_addr;
1834         dram_range_end = prop->dmmu.end_addr;
1835         dram_page_size = prop->dmmu.page_size;
1836         host_range_start = prop->pmmu.start_addr;
1837         host_range_end = prop->pmmu.end_addr;
1838         host_page_size = prop->pmmu.page_size;
1839         host_huge_range_start = prop->pmmu_huge.start_addr;
1840         host_huge_range_end = prop->pmmu_huge.end_addr;
1841         host_huge_page_size = prop->pmmu_huge.page_size;
1842
1843         return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end,
1844                         host_page_size, host_huge_range_start,
1845                         host_huge_range_end, host_huge_page_size,
1846                         dram_range_start, dram_range_end, dram_page_size);
1847 }
1848
1849 /*
1850  * hl_vm_ctx_fini       - virtual memory teardown of context
1851  *
1852  * @ctx                 : pointer to the habanalabs context structure
1853  *
1854  * This function perform teardown the following:
1855  * - Virtual block list of available virtual memory
1856  * - Virtual address to area descriptor hashtable
1857  * - MMU for context
1858  *
1859  * In addition this function does the following:
1860  * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
1861  *   hashtable should be empty as no valid mappings should exist at this
1862  *   point.
1863  * - Frees any existing physical page list from the idr which relates to the
1864  *   current context asid.
1865  * - This function checks the virtual block list for correctness. At this point
1866  *   the list should contain one element which describes the whole virtual
1867  *   memory range of the context. Otherwise, a warning is printed.
1868  */
1869 void hl_vm_ctx_fini(struct hl_ctx *ctx)
1870 {
1871         struct hl_device *hdev = ctx->hdev;
1872         struct hl_vm *vm = &hdev->vm;
1873         struct hl_vm_phys_pg_pack *phys_pg_list;
1874         struct hl_vm_hash_node *hnode;
1875         struct hlist_node *tmp_node;
1876         int i;
1877
1878         if (!ctx->hdev->mmu_enable)
1879                 return;
1880
1881         hl_debugfs_remove_ctx_mem_hash(hdev, ctx);
1882
1883         /*
1884          * Clearly something went wrong on hard reset so no point in printing
1885          * another side effect error
1886          */
1887         if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
1888                 dev_notice(hdev->dev,
1889                         "user released device without removing its memory mappings\n");
1890
1891         hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {
1892                 dev_dbg(hdev->dev,
1893                         "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n",
1894                         hnode->vaddr, ctx->asid);
1895                 unmap_device_va(ctx, hnode->vaddr, true);
1896         }
1897
1898         /* invalidate the cache once after the unmapping loop */
1899         hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
1900         hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK);
1901
1902         spin_lock(&vm->idr_lock);
1903         idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i)
1904                 if (phys_pg_list->asid == ctx->asid) {
1905                         dev_dbg(hdev->dev,
1906                                 "page list 0x%px of asid %d is still alive\n",
1907                                 phys_pg_list, ctx->asid);
1908                         atomic64_sub(phys_pg_list->total_size,
1909                                         &hdev->dram_used_mem);
1910                         free_phys_pg_pack(hdev, phys_pg_list);
1911                         idr_remove(&vm->phys_pg_pack_handles, i);
1912                 }
1913         spin_unlock(&vm->idr_lock);
1914
1915         va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]);
1916         va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]);
1917
1918         if (hdev->pmmu_huge_range)
1919                 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
1920
1921         mutex_destroy(&ctx->mem_hash_lock);
1922         hl_mmu_ctx_fini(ctx);
1923
1924         /* In this case we need to clear the global accounting of DRAM usage
1925          * because the user notifies us on allocations. If the user is no more,
1926          * all DRAM is available
1927          */
1928         if (!ctx->hdev->asic_prop.dram_supports_virtual_memory)
1929                 atomic64_set(&ctx->hdev->dram_used_mem, 0);
1930 }
1931
1932 /*
1933  * hl_vm_init           - initialize virtual memory module
1934  *
1935  * @hdev                : pointer to the habanalabs device structure
1936  *
1937  * This function initializes the following:
1938  * - MMU module
1939  * - DRAM physical pages pool of 2MB
1940  * - Idr for device memory allocation handles
1941  */
1942 int hl_vm_init(struct hl_device *hdev)
1943 {
1944         struct asic_fixed_properties *prop = &hdev->asic_prop;
1945         struct hl_vm *vm = &hdev->vm;
1946         int rc;
1947
1948         vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1);
1949         if (!vm->dram_pg_pool) {
1950                 dev_err(hdev->dev, "Failed to create dram page pool\n");
1951                 return -ENOMEM;
1952         }
1953
1954         kref_init(&vm->dram_pg_pool_refcount);
1955
1956         rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address,
1957                         prop->dram_end_address - prop->dram_user_base_address,
1958                         -1);
1959
1960         if (rc) {
1961                 dev_err(hdev->dev,
1962                         "Failed to add memory to dram page pool %d\n", rc);
1963                 goto pool_add_err;
1964         }
1965
1966         spin_lock_init(&vm->idr_lock);
1967         idr_init(&vm->phys_pg_pack_handles);
1968
1969         atomic64_set(&hdev->dram_used_mem, 0);
1970
1971         vm->init_done = true;
1972
1973         return 0;
1974
1975 pool_add_err:
1976         gen_pool_destroy(vm->dram_pg_pool);
1977
1978         return rc;
1979 }
1980
1981 /*
1982  * hl_vm_fini           - virtual memory module teardown
1983  *
1984  * @hdev                : pointer to the habanalabs device structure
1985  *
1986  * This function perform teardown to the following:
1987  * - Idr for device memory allocation handles
1988  * - DRAM physical pages pool of 2MB
1989  * - MMU module
1990  */
1991 void hl_vm_fini(struct hl_device *hdev)
1992 {
1993         struct hl_vm *vm = &hdev->vm;
1994
1995         if (!vm->init_done)
1996                 return;
1997
1998         /*
1999          * At this point all the contexts should be freed and hence no DRAM
2000          * memory should be in use. Hence the DRAM pool should be freed here.
2001          */
2002         if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1)
2003                 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n",
2004                                 __func__);
2005
2006         vm->init_done = false;
2007 }