1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
10 #include "../include/hw_ip/mmu/mmu_general.h"
12 #include <linux/uaccess.h>
13 #include <linux/slab.h>
14 #include <linux/genalloc.h>
16 #define HL_MMU_DEBUG 0
19 * The va ranges in context object contain a list with the available chunks of
20 * device virtual memory.
21 * There is one range for host allocations and one for DRAM allocations.
23 * On initialization each range contains one chunk of all of its available
24 * virtual range which is a half of the total device virtual range.
26 * On each mapping of physical pages, a suitable virtual range chunk (with a
27 * minimum size) is selected from the list. If the chunk size equals the
28 * requested size, the chunk is returned. Otherwise, the chunk is split into
29 * two chunks - one to return as result and a remainder to stay in the list.
31 * On each Unmapping of a virtual address, the relevant virtual chunk is
32 * returned to the list. The chunk is added to the list and if its edges match
33 * the edges of the adjacent chunks (means a contiguous chunk can be created),
34 * the chunks are merged.
36 * On finish, the list is checked to have only one chunk of all the relevant
37 * virtual range (which is a half of the device total virtual range).
38 * If not (means not all mappings were unmapped), a warning is printed.
42 * alloc_device_memory - allocate device memory
44 * @ctx : current context
45 * @args : host parameters containing the requested size
46 * @ret_handle : result handle
48 * This function does the following:
49 * - Allocate the requested size rounded up to 2MB pages
50 * - Return unique handle
52 static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
55 struct hl_device *hdev = ctx->hdev;
56 struct hl_vm *vm = &hdev->vm;
57 struct hl_vm_phys_pg_pack *phys_pg_pack;
58 u64 paddr = 0, total_size, num_pgs, i;
59 u32 num_curr_pgs, page_size, page_shift;
64 page_size = hdev->asic_prop.dram_page_size;
65 page_shift = __ffs(page_size);
66 num_pgs = (args->alloc.mem_size + (page_size - 1)) >> page_shift;
67 total_size = num_pgs << page_shift;
70 dev_err(hdev->dev, "Cannot allocate 0 bytes\n");
74 contiguous = args->flags & HL_MEM_CONTIGUOUS;
77 paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size);
80 "failed to allocate %llu contiguous pages with total size of %llu\n",
86 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
92 phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;
93 phys_pg_pack->asid = ctx->asid;
94 phys_pg_pack->npages = num_pgs;
95 phys_pg_pack->page_size = page_size;
96 phys_pg_pack->total_size = total_size;
97 phys_pg_pack->flags = args->flags;
98 phys_pg_pack->contiguous = contiguous;
100 phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);
101 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
106 if (phys_pg_pack->contiguous) {
107 for (i = 0 ; i < num_pgs ; i++)
108 phys_pg_pack->pages[i] = paddr + i * page_size;
110 for (i = 0 ; i < num_pgs ; i++) {
111 phys_pg_pack->pages[i] = (u64) gen_pool_alloc(
114 if (!phys_pg_pack->pages[i]) {
116 "Failed to allocate device memory (out of memory)\n");
125 spin_lock(&vm->idr_lock);
126 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
128 spin_unlock(&vm->idr_lock);
131 dev_err(hdev->dev, "Failed to get handle for page\n");
136 for (i = 0 ; i < num_pgs ; i++)
137 kref_get(&vm->dram_pg_pool_refcount);
139 phys_pg_pack->handle = handle;
141 atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);
142 atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);
144 *ret_handle = handle;
150 if (!phys_pg_pack->contiguous)
151 for (i = 0 ; i < num_curr_pgs ; i++)
152 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],
155 kvfree(phys_pg_pack->pages);
160 gen_pool_free(vm->dram_pg_pool, paddr, total_size);
166 * dma_map_host_va - DMA mapping of the given host virtual address.
167 * @hdev: habanalabs device structure
168 * @addr: the host virtual address of the memory area
169 * @size: the size of the memory area
170 * @p_userptr: pointer to result userptr structure
172 * This function does the following:
173 * - Allocate userptr structure
174 * - Pin the given host memory using the userptr structure
175 * - Perform DMA mapping to have the DMA addresses of the pages
177 static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
178 struct hl_userptr **p_userptr)
180 struct hl_userptr *userptr;
183 userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
189 rc = hl_pin_host_memory(hdev, addr, size, userptr);
191 dev_err(hdev->dev, "Failed to pin host memory\n");
195 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
196 userptr->sgt->nents, DMA_BIDIRECTIONAL);
198 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
202 userptr->dma_mapped = true;
203 userptr->dir = DMA_BIDIRECTIONAL;
204 userptr->vm_type = VM_TYPE_USERPTR;
206 *p_userptr = userptr;
211 hl_unpin_host_memory(hdev, userptr);
220 * dma_unmap_host_va - DMA unmapping of the given host virtual address.
221 * @hdev: habanalabs device structure
222 * @userptr: userptr to free
224 * This function does the following:
225 * - Unpins the physical pages
226 * - Frees the userptr structure
228 static void dma_unmap_host_va(struct hl_device *hdev,
229 struct hl_userptr *userptr)
231 hl_unpin_host_memory(hdev, userptr);
236 * dram_pg_pool_do_release - free DRAM pages pool
238 * @ref : pointer to reference object
240 * This function does the following:
241 * - Frees the idr structure of physical pages handles
242 * - Frees the generic pool of DRAM physical pages
244 static void dram_pg_pool_do_release(struct kref *ref)
246 struct hl_vm *vm = container_of(ref, struct hl_vm,
247 dram_pg_pool_refcount);
250 * free the idr here as only here we know for sure that there are no
251 * allocated physical pages and hence there are no handles in use
253 idr_destroy(&vm->phys_pg_pack_handles);
254 gen_pool_destroy(vm->dram_pg_pool);
258 * free_phys_pg_pack - free physical page pack
259 * @hdev: habanalabs device structure
260 * @phys_pg_pack: physical page pack to free
262 * This function does the following:
263 * - For DRAM memory only, iterate over the pack and free each physical block
264 * structure by returning it to the general pool
265 * - Free the hl_vm_phys_pg_pack structure
267 static void free_phys_pg_pack(struct hl_device *hdev,
268 struct hl_vm_phys_pg_pack *phys_pg_pack)
270 struct hl_vm *vm = &hdev->vm;
273 if (!phys_pg_pack->created_from_userptr) {
274 if (phys_pg_pack->contiguous) {
275 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],
276 phys_pg_pack->total_size);
278 for (i = 0; i < phys_pg_pack->npages ; i++)
279 kref_put(&vm->dram_pg_pool_refcount,
280 dram_pg_pool_do_release);
282 for (i = 0 ; i < phys_pg_pack->npages ; i++) {
283 gen_pool_free(vm->dram_pg_pool,
284 phys_pg_pack->pages[i],
285 phys_pg_pack->page_size);
286 kref_put(&vm->dram_pg_pool_refcount,
287 dram_pg_pool_do_release);
292 kvfree(phys_pg_pack->pages);
297 * free_device_memory - free device memory
299 * @ctx : current context
300 * @handle : handle of the memory chunk to free
302 * This function does the following:
303 * - Free the device memory related to the given handle
305 static int free_device_memory(struct hl_ctx *ctx, u32 handle)
307 struct hl_device *hdev = ctx->hdev;
308 struct hl_vm *vm = &hdev->vm;
309 struct hl_vm_phys_pg_pack *phys_pg_pack;
311 spin_lock(&vm->idr_lock);
312 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
314 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {
315 dev_err(hdev->dev, "handle %u is mapped, cannot free\n",
317 spin_unlock(&vm->idr_lock);
322 * must remove from idr before the freeing of the physical
323 * pages as the refcount of the pool is also the trigger of the
326 idr_remove(&vm->phys_pg_pack_handles, handle);
327 spin_unlock(&vm->idr_lock);
329 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);
330 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
332 free_phys_pg_pack(hdev, phys_pg_pack);
334 spin_unlock(&vm->idr_lock);
336 "free device memory failed, no match for handle %u\n",
345 * clear_va_list_locked - free virtual addresses list
347 * @hdev : habanalabs device structure
348 * @va_list : list of virtual addresses to free
350 * This function does the following:
351 * - Iterate over the list and free each virtual addresses block
353 * This function should be called only when va_list lock is taken
355 static void clear_va_list_locked(struct hl_device *hdev,
356 struct list_head *va_list)
358 struct hl_vm_va_block *va_block, *tmp;
360 list_for_each_entry_safe(va_block, tmp, va_list, node) {
361 list_del(&va_block->node);
367 * print_va_list_locked - print virtual addresses list
369 * @hdev : habanalabs device structure
370 * @va_list : list of virtual addresses to print
372 * This function does the following:
373 * - Iterate over the list and print each virtual addresses block
375 * This function should be called only when va_list lock is taken
377 static void print_va_list_locked(struct hl_device *hdev,
378 struct list_head *va_list)
381 struct hl_vm_va_block *va_block;
383 dev_dbg(hdev->dev, "print va list:\n");
385 list_for_each_entry(va_block, va_list, node)
387 "va block, start: 0x%llx, end: 0x%llx, size: %llu\n",
388 va_block->start, va_block->end, va_block->size);
393 * merge_va_blocks_locked - merge a virtual block if possible
395 * @hdev : pointer to the habanalabs device structure
396 * @va_list : pointer to the virtual addresses block list
397 * @va_block : virtual block to merge with adjacent blocks
399 * This function does the following:
400 * - Merge the given blocks with the adjacent blocks if their virtual ranges
401 * create a contiguous virtual range
403 * This Function should be called only when va_list lock is taken
405 static void merge_va_blocks_locked(struct hl_device *hdev,
406 struct list_head *va_list, struct hl_vm_va_block *va_block)
408 struct hl_vm_va_block *prev, *next;
410 prev = list_prev_entry(va_block, node);
411 if (&prev->node != va_list && prev->end + 1 == va_block->start) {
412 prev->end = va_block->end;
413 prev->size = prev->end - prev->start;
414 list_del(&va_block->node);
419 next = list_next_entry(va_block, node);
420 if (&next->node != va_list && va_block->end + 1 == next->start) {
421 next->start = va_block->start;
422 next->size = next->end - next->start;
423 list_del(&va_block->node);
429 * add_va_block_locked - add a virtual block to the virtual addresses list
431 * @hdev : pointer to the habanalabs device structure
432 * @va_list : pointer to the virtual addresses block list
433 * @start : start virtual address
434 * @end : end virtual address
436 * This function does the following:
437 * - Add the given block to the virtual blocks list and merge with other
438 * blocks if a contiguous virtual block can be created
440 * This Function should be called only when va_list lock is taken
442 static int add_va_block_locked(struct hl_device *hdev,
443 struct list_head *va_list, u64 start, u64 end)
445 struct hl_vm_va_block *va_block, *res = NULL;
446 u64 size = end - start;
448 print_va_list_locked(hdev, va_list);
450 list_for_each_entry(va_block, va_list, node) {
451 /* TODO: remove upon matureness */
452 if (hl_mem_area_crosses_range(start, size, va_block->start,
455 "block crossing ranges at start 0x%llx, end 0x%llx\n",
456 va_block->start, va_block->end);
460 if (va_block->end < start)
464 va_block = kmalloc(sizeof(*va_block), GFP_KERNEL);
468 va_block->start = start;
470 va_block->size = size;
473 list_add(&va_block->node, va_list);
475 list_add(&va_block->node, &res->node);
477 merge_va_blocks_locked(hdev, va_list, va_block);
479 print_va_list_locked(hdev, va_list);
485 * add_va_block - wrapper for add_va_block_locked
487 * @hdev : pointer to the habanalabs device structure
488 * @va_list : pointer to the virtual addresses block list
489 * @start : start virtual address
490 * @end : end virtual address
492 * This function does the following:
493 * - Takes the list lock and calls add_va_block_locked
495 static inline int add_va_block(struct hl_device *hdev,
496 struct hl_va_range *va_range, u64 start, u64 end)
500 mutex_lock(&va_range->lock);
501 rc = add_va_block_locked(hdev, &va_range->list, start, end);
502 mutex_unlock(&va_range->lock);
508 * get_va_block() - get a virtual block for the given size and alignment.
509 * @hdev: pointer to the habanalabs device structure.
510 * @va_range: pointer to the virtual addresses range.
511 * @size: requested block size.
512 * @hint_addr: hint for requested address by the user.
513 * @va_block_align: required alignment of the virtual block start address.
515 * This function does the following:
516 * - Iterate on the virtual block list to find a suitable virtual block for the
517 * given size and alignment.
518 * - Reserve the requested block and update the list.
519 * - Return the start address of the virtual block.
521 static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range,
522 u64 size, u64 hint_addr, u32 va_block_align)
524 struct hl_vm_va_block *va_block, *new_va_block = NULL;
525 u64 valid_start, valid_size, prev_start, prev_end, align_mask,
526 res_valid_start = 0, res_valid_size = 0;
527 bool add_prev = false;
529 align_mask = ~((u64)va_block_align - 1);
531 /* check if hint_addr is aligned */
532 if (hint_addr & (va_block_align - 1))
535 mutex_lock(&va_range->lock);
537 print_va_list_locked(hdev, &va_range->list);
539 list_for_each_entry(va_block, &va_range->list, node) {
540 /* calc the first possible aligned addr */
541 valid_start = va_block->start;
543 if (valid_start & (va_block_align - 1)) {
544 valid_start &= align_mask;
545 valid_start += va_block_align;
546 if (valid_start > va_block->end)
550 valid_size = va_block->end - valid_start;
552 if (valid_size >= size &&
553 (!new_va_block || valid_size < res_valid_size)) {
554 new_va_block = va_block;
555 res_valid_start = valid_start;
556 res_valid_size = valid_size;
559 if (hint_addr && hint_addr >= valid_start &&
560 ((hint_addr + size) <= va_block->end)) {
561 new_va_block = va_block;
562 res_valid_start = hint_addr;
563 res_valid_size = valid_size;
569 dev_err(hdev->dev, "no available va block for size %llu\n",
574 if (res_valid_start > new_va_block->start) {
575 prev_start = new_va_block->start;
576 prev_end = res_valid_start - 1;
578 new_va_block->start = res_valid_start;
579 new_va_block->size = res_valid_size;
584 if (new_va_block->size > size) {
585 new_va_block->start += size;
586 new_va_block->size = new_va_block->end - new_va_block->start;
588 list_del(&new_va_block->node);
593 add_va_block_locked(hdev, &va_range->list, prev_start,
596 print_va_list_locked(hdev, &va_range->list);
598 mutex_unlock(&va_range->lock);
600 return res_valid_start;
604 * get_sg_info - get number of pages and the DMA address from SG list
607 * @dma_addr : pointer to DMA address to return
609 * Calculate the number of consecutive pages described by the SG list. Take the
610 * offset of the address in the first page, add to it the length and round it up
611 * to the number of needed pages.
613 static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
615 *dma_addr = sg_dma_address(sg);
617 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
618 (PAGE_SIZE - 1)) >> PAGE_SHIFT;
622 * init_phys_pg_pack_from_userptr - initialize physical page pack from host
624 * @ctx: current context
625 * @userptr: userptr to initialize from
626 * @pphys_pg_pack: result pointer
628 * This function does the following:
629 * - Pin the physical pages related to the given virtual block
630 * - Create a physical page pack from the physical pages related to the given
633 static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
634 struct hl_userptr *userptr,
635 struct hl_vm_phys_pg_pack **pphys_pg_pack)
637 struct hl_vm_phys_pg_pack *phys_pg_pack;
638 struct scatterlist *sg;
640 u64 page_mask, total_npages;
641 u32 npages, page_size = PAGE_SIZE,
642 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
643 bool first = true, is_huge_page_opt = true;
645 u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
647 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
651 phys_pg_pack->vm_type = userptr->vm_type;
652 phys_pg_pack->created_from_userptr = true;
653 phys_pg_pack->asid = ctx->asid;
654 atomic_set(&phys_pg_pack->mapping_cnt, 1);
656 /* Only if all dma_addrs are aligned to 2MB and their
657 * sizes is at least 2MB, we can use huge page mapping.
658 * We limit the 2MB optimization to this condition,
659 * since later on we acquire the related VA range as one
663 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
664 npages = get_sg_info(sg, &dma_addr);
666 total_npages += npages;
668 if ((npages % pgs_in_huge_page) ||
669 (dma_addr & (huge_page_size - 1)))
670 is_huge_page_opt = false;
673 if (is_huge_page_opt) {
674 page_size = huge_page_size;
675 do_div(total_npages, pgs_in_huge_page);
678 page_mask = ~(((u64) page_size) - 1);
680 phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),
682 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
684 goto page_pack_arr_mem_err;
687 phys_pg_pack->npages = total_npages;
688 phys_pg_pack->page_size = page_size;
689 phys_pg_pack->total_size = total_npages * page_size;
692 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
693 npages = get_sg_info(sg, &dma_addr);
695 /* align down to physical page size and save the offset */
698 phys_pg_pack->offset = dma_addr & (page_size - 1);
699 dma_addr &= page_mask;
703 phys_pg_pack->pages[j++] = dma_addr;
704 dma_addr += page_size;
706 if (is_huge_page_opt)
707 npages -= pgs_in_huge_page;
713 *pphys_pg_pack = phys_pg_pack;
717 page_pack_arr_mem_err:
724 * map_phys_pg_pack - maps the physical page pack.
725 * @ctx: current context
726 * @vaddr: start address of the virtual area to map from
727 * @phys_pg_pack: the pack of physical pages to map to
729 * This function does the following:
730 * - Maps each chunk of virtual memory to matching physical chunk
731 * - Stores number of successful mappings in the given argument
732 * - Returns 0 on success, error code otherwise
734 static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
735 struct hl_vm_phys_pg_pack *phys_pg_pack)
737 struct hl_device *hdev = ctx->hdev;
738 u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;
739 u32 page_size = phys_pg_pack->page_size;
742 for (i = 0 ; i < phys_pg_pack->npages ; i++) {
743 paddr = phys_pg_pack->pages[i];
745 rc = hl_mmu_map(ctx, next_vaddr, paddr, page_size,
746 (i + 1) == phys_pg_pack->npages);
749 "map failed for handle %u, npages: %llu, mapped: %llu",
750 phys_pg_pack->handle, phys_pg_pack->npages,
756 next_vaddr += page_size;
763 for (i = 0 ; i < mapped_pg_cnt ; i++) {
764 if (hl_mmu_unmap(ctx, next_vaddr, page_size,
765 (i + 1) == mapped_pg_cnt))
766 dev_warn_ratelimited(hdev->dev,
767 "failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n",
768 phys_pg_pack->handle, next_vaddr,
769 phys_pg_pack->pages[i], page_size);
771 next_vaddr += page_size;
778 * unmap_phys_pg_pack - unmaps the physical page pack
779 * @ctx: current context
780 * @vaddr: start address of the virtual area to unmap
781 * @phys_pg_pack: the pack of physical pages to unmap
783 static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
784 struct hl_vm_phys_pg_pack *phys_pg_pack)
786 struct hl_device *hdev = ctx->hdev;
790 page_size = phys_pg_pack->page_size;
793 for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {
794 if (hl_mmu_unmap(ctx, next_vaddr, page_size,
795 (i + 1) == phys_pg_pack->npages))
796 dev_warn_ratelimited(hdev->dev,
797 "unmap failed for vaddr: 0x%llx\n", next_vaddr);
800 * unmapping on Palladium can be really long, so avoid a CPU
801 * soft lockup bug by sleeping a little between unmapping pages
804 usleep_range(500, 1000);
808 static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args,
811 struct hl_device *hdev = ctx->hdev;
812 struct hl_vm *vm = &hdev->vm;
813 struct hl_vm_phys_pg_pack *phys_pg_pack;
816 handle = lower_32_bits(args->map_device.handle);
817 spin_lock(&vm->idr_lock);
818 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
820 spin_unlock(&vm->idr_lock);
821 dev_err(hdev->dev, "no match for handle %u\n", handle);
825 *paddr = phys_pg_pack->pages[0];
827 spin_unlock(&vm->idr_lock);
833 * map_device_va - map the given memory
835 * @ctx : current context
836 * @args : host parameters with handle/host virtual address
837 * @device_addr : pointer to result device virtual address
839 * This function does the following:
840 * - If given a physical device memory handle, map to a device virtual block
841 * and return the start address of this block
842 * - If given a host virtual address and size, find the related physical pages,
843 * map a device virtual block to this pages and return the start address of
846 static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
849 struct hl_device *hdev = ctx->hdev;
850 struct hl_vm *vm = &hdev->vm;
851 struct hl_vm_phys_pg_pack *phys_pg_pack;
852 struct hl_userptr *userptr = NULL;
853 struct hl_vm_hash_node *hnode;
854 struct hl_va_range *va_range;
855 enum vm_type_t *vm_type;
856 u64 ret_vaddr, hint_addr;
857 u32 handle = 0, va_block_align;
859 bool is_userptr = args->flags & HL_MEM_USERPTR;
865 u64 addr = args->map_host.host_virt_addr,
866 size = args->map_host.mem_size;
867 u32 page_size = hdev->asic_prop.pmmu.page_size,
868 huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
870 rc = dma_map_host_va(hdev, addr, size, &userptr);
872 dev_err(hdev->dev, "failed to get userptr from va\n");
876 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
880 "unable to init page pack for vaddr 0x%llx\n",
882 goto init_page_pack_err;
885 vm_type = (enum vm_type_t *) userptr;
886 hint_addr = args->map_host.hint_addr;
887 handle = phys_pg_pack->handle;
889 /* get required alignment */
890 if (phys_pg_pack->page_size == page_size) {
891 va_range = ctx->host_va_range;
894 * huge page alignment may be needed in case of regular
895 * page mapping, depending on the host VA alignment
897 if (addr & (huge_page_size - 1))
898 va_block_align = page_size;
900 va_block_align = huge_page_size;
903 * huge page alignment is needed in case of huge page
906 va_range = ctx->host_huge_va_range;
907 va_block_align = huge_page_size;
910 handle = lower_32_bits(args->map_device.handle);
912 spin_lock(&vm->idr_lock);
913 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
915 spin_unlock(&vm->idr_lock);
917 "no match for handle %u\n", handle);
921 /* increment now to avoid freeing device memory while mapping */
922 atomic_inc(&phys_pg_pack->mapping_cnt);
924 spin_unlock(&vm->idr_lock);
926 vm_type = (enum vm_type_t *) phys_pg_pack;
928 hint_addr = args->map_device.hint_addr;
930 /* DRAM VA alignment is the same as the DRAM page size */
931 va_range = ctx->dram_va_range;
932 va_block_align = hdev->asic_prop.dmmu.page_size;
936 * relevant for mapping device physical memory only, as host memory is
939 if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&
940 phys_pg_pack->asid != ctx->asid) {
942 "Failed to map memory, handle %u is not shared\n",
948 hnode = kzalloc(sizeof(*hnode), GFP_KERNEL);
954 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
955 hint_addr, va_block_align);
957 dev_err(hdev->dev, "no available va block for handle %u\n",
963 mutex_lock(&ctx->mmu_lock);
965 rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
967 mutex_unlock(&ctx->mmu_lock);
968 dev_err(hdev->dev, "mapping page pack failed for handle %u\n",
973 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
975 mutex_unlock(&ctx->mmu_lock);
979 "mapping handle %u failed due to MMU cache invalidation\n",
984 ret_vaddr += phys_pg_pack->offset;
986 hnode->ptr = vm_type;
987 hnode->vaddr = ret_vaddr;
989 mutex_lock(&ctx->mem_hash_lock);
990 hash_add(ctx->mem_hash, &hnode->node, ret_vaddr);
991 mutex_unlock(&ctx->mem_hash_lock);
993 *device_addr = ret_vaddr;
996 free_phys_pg_pack(hdev, phys_pg_pack);
1001 if (add_va_block(hdev, va_range, ret_vaddr,
1002 ret_vaddr + phys_pg_pack->total_size - 1))
1004 "release va block failed for handle 0x%x, vaddr: 0x%llx\n",
1011 atomic_dec(&phys_pg_pack->mapping_cnt);
1013 free_phys_pg_pack(hdev, phys_pg_pack);
1016 dma_unmap_host_va(hdev, userptr);
1022 * unmap_device_va - unmap the given device virtual address
1024 * @ctx : current context
1025 * @vaddr : device virtual address to unmap
1026 * @ctx_free : true if in context free flow, false otherwise.
1028 * This function does the following:
1029 * - Unmap the physical pages related to the given virtual address
1030 * - return the device virtual block to the virtual block list
1032 static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
1034 struct hl_device *hdev = ctx->hdev;
1035 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
1036 struct hl_vm_hash_node *hnode = NULL;
1037 struct hl_userptr *userptr = NULL;
1038 struct hl_va_range *va_range;
1039 enum vm_type_t *vm_type;
1043 /* protect from double entrance */
1044 mutex_lock(&ctx->mem_hash_lock);
1045 hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
1046 if (vaddr == hnode->vaddr)
1050 mutex_unlock(&ctx->mem_hash_lock);
1052 "unmap failed, no mem hnode for vaddr 0x%llx\n",
1057 hash_del(&hnode->node);
1058 mutex_unlock(&ctx->mem_hash_lock);
1060 vm_type = hnode->ptr;
1062 if (*vm_type == VM_TYPE_USERPTR) {
1064 userptr = hnode->ptr;
1065 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1069 "unable to init page pack for vaddr 0x%llx\n",
1074 if (phys_pg_pack->page_size ==
1075 hdev->asic_prop.pmmu.page_size)
1076 va_range = ctx->host_va_range;
1078 va_range = ctx->host_huge_va_range;
1079 } else if (*vm_type == VM_TYPE_PHYS_PACK) {
1081 va_range = ctx->dram_va_range;
1082 phys_pg_pack = hnode->ptr;
1085 "unmap failed, unknown vm desc for vaddr 0x%llx\n",
1091 if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {
1092 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr);
1094 goto mapping_cnt_err;
1097 vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);
1099 mutex_lock(&ctx->mmu_lock);
1101 unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);
1104 * During context free this function is called in a loop to clean all
1105 * the context mappings. Hence the cache invalidation can be called once
1106 * at the loop end rather than for each iteration
1109 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
1112 mutex_unlock(&ctx->mmu_lock);
1115 * If the context is closing we don't need to check for the MMU cache
1116 * invalidation return code and update the VA free list as in this flow
1117 * we invalidate the MMU cache outside of this unmap function and the VA
1118 * free list will be freed anyway.
1125 "unmapping vaddr 0x%llx failed due to MMU cache invalidation\n",
1128 tmp_rc = add_va_block(hdev, va_range, vaddr,
1129 vaddr + phys_pg_pack->total_size - 1);
1132 "add va block failed for vaddr: 0x%llx\n",
1139 atomic_dec(&phys_pg_pack->mapping_cnt);
1143 free_phys_pg_pack(hdev, phys_pg_pack);
1144 dma_unmap_host_va(hdev, userptr);
1151 free_phys_pg_pack(hdev, phys_pg_pack);
1153 mutex_lock(&ctx->mem_hash_lock);
1154 hash_add(ctx->mem_hash, &hnode->node, vaddr);
1155 mutex_unlock(&ctx->mem_hash_lock);
1160 static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args)
1162 struct hl_device *hdev = hpriv->hdev;
1163 struct hl_ctx *ctx = hpriv->ctx;
1164 u64 device_addr = 0;
1168 switch (args->in.op) {
1169 case HL_MEM_OP_ALLOC:
1170 if (args->in.alloc.mem_size == 0) {
1172 "alloc size must be larger than 0\n");
1177 /* Force contiguous as there are no real MMU
1178 * translations to overcome physical memory gaps
1180 args->in.flags |= HL_MEM_CONTIGUOUS;
1181 rc = alloc_device_memory(ctx, &args->in, &handle);
1183 memset(args, 0, sizeof(*args));
1184 args->out.handle = (__u64) handle;
1187 case HL_MEM_OP_FREE:
1188 rc = free_device_memory(ctx, args->in.free.handle);
1192 if (args->in.flags & HL_MEM_USERPTR) {
1193 device_addr = args->in.map_host.host_virt_addr;
1196 rc = get_paddr_from_handle(ctx, &args->in,
1200 memset(args, 0, sizeof(*args));
1201 args->out.device_virt_addr = device_addr;
1204 case HL_MEM_OP_UNMAP:
1209 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1218 int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
1220 union hl_mem_args *args = data;
1221 struct hl_device *hdev = hpriv->hdev;
1222 struct hl_ctx *ctx = hpriv->ctx;
1223 u64 device_addr = 0;
1227 if (hl_device_disabled_or_in_reset(hdev)) {
1228 dev_warn_ratelimited(hdev->dev,
1229 "Device is %s. Can't execute MEMORY IOCTL\n",
1230 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
1234 if (!hdev->mmu_enable)
1235 return mem_ioctl_no_mmu(hpriv, args);
1237 switch (args->in.op) {
1238 case HL_MEM_OP_ALLOC:
1239 if (!hdev->dram_supports_virtual_memory) {
1240 dev_err(hdev->dev, "DRAM alloc is not supported\n");
1245 if (args->in.alloc.mem_size == 0) {
1247 "alloc size must be larger than 0\n");
1251 rc = alloc_device_memory(ctx, &args->in, &handle);
1253 memset(args, 0, sizeof(*args));
1254 args->out.handle = (__u64) handle;
1257 case HL_MEM_OP_FREE:
1258 rc = free_device_memory(ctx, args->in.free.handle);
1262 rc = map_device_va(ctx, &args->in, &device_addr);
1264 memset(args, 0, sizeof(*args));
1265 args->out.device_virt_addr = device_addr;
1268 case HL_MEM_OP_UNMAP:
1269 rc = unmap_device_va(ctx, args->in.unmap.device_virt_addr,
1274 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1283 static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
1284 u32 npages, u64 start, u32 offset,
1285 struct hl_userptr *userptr)
1289 if (!access_ok((void __user *) (uintptr_t) addr, size)) {
1290 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr);
1294 userptr->vec = frame_vector_create(npages);
1295 if (!userptr->vec) {
1296 dev_err(hdev->dev, "Failed to create frame vector\n");
1300 rc = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
1305 "Failed to map host memory, user ptr probably wrong\n");
1307 goto destroy_framevec;
1312 if (frame_vector_to_pages(userptr->vec) < 0) {
1314 "Failed to translate frame vector to pages\n");
1319 rc = sg_alloc_table_from_pages(userptr->sgt,
1320 frame_vector_pages(userptr->vec),
1321 npages, offset, size, GFP_ATOMIC);
1323 dev_err(hdev->dev, "failed to create SG table from pages\n");
1330 put_vaddr_frames(userptr->vec);
1332 frame_vector_destroy(userptr->vec);
1337 * hl_pin_host_memory - pins a chunk of host memory.
1338 * @hdev: pointer to the habanalabs device structure
1339 * @addr: the host virtual address of the memory area
1340 * @size: the size of the memory area
1341 * @userptr: pointer to hl_userptr structure
1343 * This function does the following:
1344 * - Pins the physical pages
1345 * - Create an SG list from those pages
1347 int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1348 struct hl_userptr *userptr)
1355 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size);
1360 * If the combination of the address and size requested for this memory
1361 * region causes an integer overflow, return error.
1363 if (((addr + size) < addr) ||
1364 PAGE_ALIGN(addr + size) < (addr + size)) {
1366 "user pointer 0x%llx + %llu causes integer overflow\n",
1372 * This function can be called also from data path, hence use atomic
1373 * always as it is not a big allocation.
1375 userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_ATOMIC);
1379 start = addr & PAGE_MASK;
1380 offset = addr & ~PAGE_MASK;
1381 end = PAGE_ALIGN(addr + size);
1382 npages = (end - start) >> PAGE_SHIFT;
1384 userptr->size = size;
1385 userptr->addr = addr;
1386 userptr->dma_mapped = false;
1387 INIT_LIST_HEAD(&userptr->job_node);
1389 rc = get_user_memory(hdev, addr, size, npages, start, offset,
1393 "failed to get user memory for address 0x%llx\n",
1398 hl_debugfs_add_userptr(hdev, userptr);
1403 kfree(userptr->sgt);
1408 * hl_unpin_host_memory - unpins a chunk of host memory.
1409 * @hdev: pointer to the habanalabs device structure
1410 * @userptr: pointer to hl_userptr structure
1412 * This function does the following:
1413 * - Unpins the physical pages related to the host memory
1414 * - Free the SG list
1416 void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
1418 struct page **pages;
1420 hl_debugfs_remove_userptr(hdev, userptr);
1422 if (userptr->dma_mapped)
1423 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl,
1424 userptr->sgt->nents,
1427 pages = frame_vector_pages(userptr->vec);
1428 if (!IS_ERR(pages)) {
1431 for (i = 0; i < frame_vector_count(userptr->vec); i++)
1432 set_page_dirty_lock(pages[i]);
1434 put_vaddr_frames(userptr->vec);
1435 frame_vector_destroy(userptr->vec);
1437 list_del(&userptr->job_node);
1439 sg_free_table(userptr->sgt);
1440 kfree(userptr->sgt);
1444 * hl_userptr_delete_list - clear userptr list
1446 * @hdev : pointer to the habanalabs device structure
1447 * @userptr_list : pointer to the list to clear
1449 * This function does the following:
1450 * - Iterates over the list and unpins the host memory and frees the userptr
1453 void hl_userptr_delete_list(struct hl_device *hdev,
1454 struct list_head *userptr_list)
1456 struct hl_userptr *userptr, *tmp;
1458 list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) {
1459 hl_unpin_host_memory(hdev, userptr);
1463 INIT_LIST_HEAD(userptr_list);
1467 * hl_userptr_is_pinned - returns whether the given userptr is pinned
1469 * @hdev : pointer to the habanalabs device structure
1470 * @userptr_list : pointer to the list to clear
1471 * @userptr : pointer to userptr to check
1473 * This function does the following:
1474 * - Iterates over the list and checks if the given userptr is in it, means is
1475 * pinned. If so, returns true, otherwise returns false.
1477 bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
1478 u32 size, struct list_head *userptr_list,
1479 struct hl_userptr **userptr)
1481 list_for_each_entry((*userptr), userptr_list, job_node) {
1482 if ((addr == (*userptr)->addr) && (size == (*userptr)->size))
1490 * va_range_init - initialize virtual addresses range
1491 * @hdev: pointer to the habanalabs device structure
1492 * @va_range: pointer to the range to initialize
1493 * @start: range start address
1494 * @end: range end address
1496 * This function does the following:
1497 * - Initializes the virtual addresses list of the given range with the given
1500 static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
1505 INIT_LIST_HEAD(&va_range->list);
1507 /* PAGE_SIZE alignment */
1509 if (start & (PAGE_SIZE - 1)) {
1514 if (end & (PAGE_SIZE - 1))
1518 dev_err(hdev->dev, "too small vm range for va list\n");
1522 rc = add_va_block(hdev, va_range, start, end);
1525 dev_err(hdev->dev, "Failed to init host va list\n");
1529 va_range->start_addr = start;
1530 va_range->end_addr = end;
1536 * va_range_fini() - clear a virtual addresses range
1537 * @hdev: pointer to the habanalabs structure
1538 * va_range: pointer to virtual addresses range
1540 * This function does the following:
1541 * - Frees the virtual addresses block list and its lock
1543 static void va_range_fini(struct hl_device *hdev,
1544 struct hl_va_range *va_range)
1546 mutex_lock(&va_range->lock);
1547 clear_va_list_locked(hdev, &va_range->list);
1548 mutex_unlock(&va_range->lock);
1550 mutex_destroy(&va_range->lock);
1555 * vm_ctx_init_with_ranges() - initialize virtual memory for context
1556 * @ctx: pointer to the habanalabs context structure
1557 * @host_range_start: host virtual addresses range start.
1558 * @host_range_end: host virtual addresses range end.
1559 * @host_huge_range_start: host virtual addresses range start for memory
1560 * allocated with huge pages.
1561 * @host_huge_range_end: host virtual addresses range end for memory allocated
1563 * @dram_range_start: dram virtual addresses range start.
1564 * @dram_range_end: dram virtual addresses range end.
1566 * This function initializes the following:
1568 * - Virtual address to area descriptor hashtable
1569 * - Virtual block list of available virtual memory
1571 static int vm_ctx_init_with_ranges(struct hl_ctx *ctx,
1572 u64 host_range_start,
1574 u64 host_huge_range_start,
1575 u64 host_huge_range_end,
1576 u64 dram_range_start,
1579 struct hl_device *hdev = ctx->hdev;
1582 ctx->host_va_range = kzalloc(sizeof(*ctx->host_va_range), GFP_KERNEL);
1583 if (!ctx->host_va_range)
1586 ctx->host_huge_va_range = kzalloc(sizeof(*ctx->host_huge_va_range),
1588 if (!ctx->host_huge_va_range) {
1590 goto host_huge_va_range_err;
1593 ctx->dram_va_range = kzalloc(sizeof(*ctx->dram_va_range), GFP_KERNEL);
1594 if (!ctx->dram_va_range) {
1596 goto dram_va_range_err;
1599 rc = hl_mmu_ctx_init(ctx);
1601 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
1605 mutex_init(&ctx->mem_hash_lock);
1606 hash_init(ctx->mem_hash);
1608 mutex_init(&ctx->host_va_range->lock);
1610 rc = va_range_init(hdev, ctx->host_va_range, host_range_start,
1613 dev_err(hdev->dev, "failed to init host vm range\n");
1614 goto host_page_range_err;
1617 if (hdev->pmmu_huge_range) {
1618 mutex_init(&ctx->host_huge_va_range->lock);
1620 rc = va_range_init(hdev, ctx->host_huge_va_range,
1621 host_huge_range_start,
1622 host_huge_range_end);
1625 "failed to init host huge vm range\n");
1626 goto host_hpage_range_err;
1629 ctx->host_huge_va_range = ctx->host_va_range;
1632 mutex_init(&ctx->dram_va_range->lock);
1634 rc = va_range_init(hdev, ctx->dram_va_range, dram_range_start,
1637 dev_err(hdev->dev, "failed to init dram vm range\n");
1641 hl_debugfs_add_ctx_mem_hash(hdev, ctx);
1646 mutex_destroy(&ctx->dram_va_range->lock);
1648 if (hdev->pmmu_huge_range) {
1649 mutex_lock(&ctx->host_huge_va_range->lock);
1650 clear_va_list_locked(hdev, &ctx->host_huge_va_range->list);
1651 mutex_unlock(&ctx->host_huge_va_range->lock);
1653 host_hpage_range_err:
1654 if (hdev->pmmu_huge_range)
1655 mutex_destroy(&ctx->host_huge_va_range->lock);
1656 mutex_lock(&ctx->host_va_range->lock);
1657 clear_va_list_locked(hdev, &ctx->host_va_range->list);
1658 mutex_unlock(&ctx->host_va_range->lock);
1659 host_page_range_err:
1660 mutex_destroy(&ctx->host_va_range->lock);
1661 mutex_destroy(&ctx->mem_hash_lock);
1662 hl_mmu_ctx_fini(ctx);
1664 kfree(ctx->dram_va_range);
1666 kfree(ctx->host_huge_va_range);
1667 host_huge_va_range_err:
1668 kfree(ctx->host_va_range);
1673 int hl_vm_ctx_init(struct hl_ctx *ctx)
1675 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
1676 u64 host_range_start, host_range_end, host_huge_range_start,
1677 host_huge_range_end, dram_range_start, dram_range_end;
1679 atomic64_set(&ctx->dram_phys_mem, 0);
1682 * - If MMU is enabled, init the ranges as usual.
1683 * - If MMU is disabled, in case of host mapping, the returned address
1685 * In case of DRAM mapping, the returned address is the physical
1686 * address of the memory related to the given handle.
1688 if (ctx->hdev->mmu_enable) {
1689 dram_range_start = prop->dmmu.start_addr;
1690 dram_range_end = prop->dmmu.end_addr;
1691 host_range_start = prop->pmmu.start_addr;
1692 host_range_end = prop->pmmu.end_addr;
1693 host_huge_range_start = prop->pmmu_huge.start_addr;
1694 host_huge_range_end = prop->pmmu_huge.end_addr;
1696 dram_range_start = prop->dram_user_base_address;
1697 dram_range_end = prop->dram_end_address;
1698 host_range_start = prop->dram_user_base_address;
1699 host_range_end = prop->dram_end_address;
1700 host_huge_range_start = prop->dram_user_base_address;
1701 host_huge_range_end = prop->dram_end_address;
1704 return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end,
1705 host_huge_range_start,
1706 host_huge_range_end,
1712 * hl_vm_ctx_fini - virtual memory teardown of context
1714 * @ctx : pointer to the habanalabs context structure
1716 * This function perform teardown the following:
1717 * - Virtual block list of available virtual memory
1718 * - Virtual address to area descriptor hashtable
1721 * In addition this function does the following:
1722 * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
1723 * hashtable should be empty as no valid mappings should exist at this
1725 * - Frees any existing physical page list from the idr which relates to the
1726 * current context asid.
1727 * - This function checks the virtual block list for correctness. At this point
1728 * the list should contain one element which describes the whole virtual
1729 * memory range of the context. Otherwise, a warning is printed.
1731 void hl_vm_ctx_fini(struct hl_ctx *ctx)
1733 struct hl_device *hdev = ctx->hdev;
1734 struct hl_vm *vm = &hdev->vm;
1735 struct hl_vm_phys_pg_pack *phys_pg_list;
1736 struct hl_vm_hash_node *hnode;
1737 struct hlist_node *tmp_node;
1740 hl_debugfs_remove_ctx_mem_hash(hdev, ctx);
1743 * Clearly something went wrong on hard reset so no point in printing
1744 * another side effect error
1746 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
1747 dev_notice(hdev->dev,
1748 "user released device without removing its memory mappings\n");
1750 hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {
1752 "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n",
1753 hnode->vaddr, ctx->asid);
1754 unmap_device_va(ctx, hnode->vaddr, true);
1757 /* invalidate the cache once after the unmapping loop */
1758 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
1759 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK);
1761 spin_lock(&vm->idr_lock);
1762 idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i)
1763 if (phys_pg_list->asid == ctx->asid) {
1765 "page list 0x%px of asid %d is still alive\n",
1766 phys_pg_list, ctx->asid);
1767 atomic64_sub(phys_pg_list->total_size,
1768 &hdev->dram_used_mem);
1769 free_phys_pg_pack(hdev, phys_pg_list);
1770 idr_remove(&vm->phys_pg_pack_handles, i);
1772 spin_unlock(&vm->idr_lock);
1774 va_range_fini(hdev, ctx->dram_va_range);
1775 if (hdev->pmmu_huge_range)
1776 va_range_fini(hdev, ctx->host_huge_va_range);
1777 va_range_fini(hdev, ctx->host_va_range);
1779 mutex_destroy(&ctx->mem_hash_lock);
1780 hl_mmu_ctx_fini(ctx);
1784 * hl_vm_init - initialize virtual memory module
1786 * @hdev : pointer to the habanalabs device structure
1788 * This function initializes the following:
1790 * - DRAM physical pages pool of 2MB
1791 * - Idr for device memory allocation handles
1793 int hl_vm_init(struct hl_device *hdev)
1795 struct asic_fixed_properties *prop = &hdev->asic_prop;
1796 struct hl_vm *vm = &hdev->vm;
1799 vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1);
1800 if (!vm->dram_pg_pool) {
1801 dev_err(hdev->dev, "Failed to create dram page pool\n");
1805 kref_init(&vm->dram_pg_pool_refcount);
1807 rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address,
1808 prop->dram_end_address - prop->dram_user_base_address,
1813 "Failed to add memory to dram page pool %d\n", rc);
1817 spin_lock_init(&vm->idr_lock);
1818 idr_init(&vm->phys_pg_pack_handles);
1820 atomic64_set(&hdev->dram_used_mem, 0);
1822 vm->init_done = true;
1827 gen_pool_destroy(vm->dram_pg_pool);
1833 * hl_vm_fini - virtual memory module teardown
1835 * @hdev : pointer to the habanalabs device structure
1837 * This function perform teardown to the following:
1838 * - Idr for device memory allocation handles
1839 * - DRAM physical pages pool of 2MB
1842 void hl_vm_fini(struct hl_device *hdev)
1844 struct hl_vm *vm = &hdev->vm;
1850 * At this point all the contexts should be freed and hence no DRAM
1851 * memory should be in use. Hence the DRAM pool should be freed here.
1853 if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1)
1854 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n",
1857 vm->init_done = false;