1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
10 #include "../include/hw_ip/mmu/mmu_general.h"
12 #include <linux/uaccess.h>
13 #include <linux/slab.h>
15 #define HL_MMU_DEBUG 0
17 /* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */
18 #define DRAM_POOL_PAGE_SIZE SZ_8M
21 * The va ranges in context object contain a list with the available chunks of
22 * device virtual memory.
23 * There is one range for host allocations and one for DRAM allocations.
25 * On initialization each range contains one chunk of all of its available
26 * virtual range which is a half of the total device virtual range.
28 * On each mapping of physical pages, a suitable virtual range chunk (with a
29 * minimum size) is selected from the list. If the chunk size equals the
30 * requested size, the chunk is returned. Otherwise, the chunk is split into
31 * two chunks - one to return as result and a remainder to stay in the list.
33 * On each Unmapping of a virtual address, the relevant virtual chunk is
34 * returned to the list. The chunk is added to the list and if its edges match
35 * the edges of the adjacent chunks (means a contiguous chunk can be created),
36 * the chunks are merged.
38 * On finish, the list is checked to have only one chunk of all the relevant
39 * virtual range (which is a half of the device total virtual range).
40 * If not (means not all mappings were unmapped), a warning is printed.
44 * alloc_device_memory() - allocate device memory.
45 * @ctx: pointer to the context structure.
46 * @args: host parameters containing the requested size.
47 * @ret_handle: result handle.
49 * This function does the following:
50 * - Allocate the requested size rounded up to 'dram_page_size' pages.
51 * - Return unique handle for later map/unmap/free.
53 static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
56 struct hl_device *hdev = ctx->hdev;
57 struct hl_vm *vm = &hdev->vm;
58 struct hl_vm_phys_pg_pack *phys_pg_pack;
59 u64 paddr = 0, total_size, num_pgs, i;
60 u32 num_curr_pgs, page_size;
65 page_size = hdev->asic_prop.dram_page_size;
66 num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size);
67 total_size = num_pgs * page_size;
70 dev_err(hdev->dev, "Cannot allocate 0 bytes\n");
74 contiguous = args->flags & HL_MEM_CONTIGUOUS;
77 paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size);
80 "failed to allocate %llu contiguous pages with total size of %llu\n",
86 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
92 phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;
93 phys_pg_pack->asid = ctx->asid;
94 phys_pg_pack->npages = num_pgs;
95 phys_pg_pack->page_size = page_size;
96 phys_pg_pack->total_size = total_size;
97 phys_pg_pack->flags = args->flags;
98 phys_pg_pack->contiguous = contiguous;
100 phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);
101 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
106 if (phys_pg_pack->contiguous) {
107 for (i = 0 ; i < num_pgs ; i++)
108 phys_pg_pack->pages[i] = paddr + i * page_size;
110 for (i = 0 ; i < num_pgs ; i++) {
111 phys_pg_pack->pages[i] = (u64) gen_pool_alloc(
114 if (!phys_pg_pack->pages[i]) {
116 "Failed to allocate device memory (out of memory)\n");
125 spin_lock(&vm->idr_lock);
126 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
128 spin_unlock(&vm->idr_lock);
131 dev_err(hdev->dev, "Failed to get handle for page\n");
136 for (i = 0 ; i < num_pgs ; i++)
137 kref_get(&vm->dram_pg_pool_refcount);
139 phys_pg_pack->handle = handle;
141 atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);
142 atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);
144 *ret_handle = handle;
150 if (!phys_pg_pack->contiguous)
151 for (i = 0 ; i < num_curr_pgs ; i++)
152 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],
155 kvfree(phys_pg_pack->pages);
160 gen_pool_free(vm->dram_pg_pool, paddr, total_size);
166 * dma_map_host_va() - DMA mapping of the given host virtual address.
167 * @hdev: habanalabs device structure.
168 * @addr: the host virtual address of the memory area.
169 * @size: the size of the memory area.
170 * @p_userptr: pointer to result userptr structure.
172 * This function does the following:
173 * - Allocate userptr structure.
174 * - Pin the given host memory using the userptr structure.
175 * - Perform DMA mapping to have the DMA addresses of the pages.
177 static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
178 struct hl_userptr **p_userptr)
180 struct hl_userptr *userptr;
183 userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
189 rc = hl_pin_host_memory(hdev, addr, size, userptr);
191 dev_err(hdev->dev, "Failed to pin host memory\n");
195 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
196 userptr->sgt->nents, DMA_BIDIRECTIONAL);
198 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
202 userptr->dma_mapped = true;
203 userptr->dir = DMA_BIDIRECTIONAL;
204 userptr->vm_type = VM_TYPE_USERPTR;
206 *p_userptr = userptr;
211 hl_unpin_host_memory(hdev, userptr);
220 * dma_unmap_host_va() - DMA unmapping of the given host virtual address.
221 * @hdev: habanalabs device structure.
222 * @userptr: userptr to free.
224 * This function does the following:
225 * - Unpins the physical pages.
226 * - Frees the userptr structure.
228 static void dma_unmap_host_va(struct hl_device *hdev,
229 struct hl_userptr *userptr)
231 hl_unpin_host_memory(hdev, userptr);
236 * dram_pg_pool_do_release() - free DRAM pages pool
237 * @ref: pointer to reference object.
239 * This function does the following:
240 * - Frees the idr structure of physical pages handles.
241 * - Frees the generic pool of DRAM physical pages.
243 static void dram_pg_pool_do_release(struct kref *ref)
245 struct hl_vm *vm = container_of(ref, struct hl_vm,
246 dram_pg_pool_refcount);
249 * free the idr here as only here we know for sure that there are no
250 * allocated physical pages and hence there are no handles in use
252 idr_destroy(&vm->phys_pg_pack_handles);
253 gen_pool_destroy(vm->dram_pg_pool);
257 * free_phys_pg_pack() - free physical page pack.
258 * @hdev: habanalabs device structure.
259 * @phys_pg_pack: physical page pack to free.
261 * This function does the following:
262 * - For DRAM memory only
263 * - iterate over the pack, scrub and free each physical block structure by
264 * returning it to the general pool.
265 * In case of error during scrubbing, initiate hard reset.
266 * Once hard reset is triggered, scrubbing is bypassed while freeing the
268 * - Free the hl_vm_phys_pg_pack structure.
270 static int free_phys_pg_pack(struct hl_device *hdev,
271 struct hl_vm_phys_pg_pack *phys_pg_pack)
273 struct hl_vm *vm = &hdev->vm;
277 if (phys_pg_pack->created_from_userptr)
280 if (phys_pg_pack->contiguous) {
281 if (hdev->memory_scrub && !hdev->disabled) {
282 rc = hdev->asic_funcs->scrub_device_mem(hdev,
283 phys_pg_pack->pages[0],
284 phys_pg_pack->total_size);
287 "Failed to scrub contiguous device memory\n");
290 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],
291 phys_pg_pack->total_size);
293 for (i = 0; i < phys_pg_pack->npages ; i++)
294 kref_put(&vm->dram_pg_pool_refcount,
295 dram_pg_pool_do_release);
297 for (i = 0 ; i < phys_pg_pack->npages ; i++) {
298 if (hdev->memory_scrub && !hdev->disabled && rc == 0) {
299 rc = hdev->asic_funcs->scrub_device_mem(
301 phys_pg_pack->pages[i],
302 phys_pg_pack->page_size);
305 "Failed to scrub device memory\n");
307 gen_pool_free(vm->dram_pg_pool,
308 phys_pg_pack->pages[i],
309 phys_pg_pack->page_size);
310 kref_put(&vm->dram_pg_pool_refcount,
311 dram_pg_pool_do_release);
315 if (rc && !hdev->disabled)
316 hl_device_reset(hdev, HL_RESET_HARD);
319 kvfree(phys_pg_pack->pages);
326 * free_device_memory() - free device memory.
327 * @ctx: pointer to the context structure.
328 * @args: host parameters containing the requested size.
330 * This function does the following:
331 * - Free the device memory related to the given handle.
333 static int free_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args)
335 struct hl_device *hdev = ctx->hdev;
336 struct hl_vm *vm = &hdev->vm;
337 struct hl_vm_phys_pg_pack *phys_pg_pack;
338 u32 handle = args->free.handle;
340 spin_lock(&vm->idr_lock);
341 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
343 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {
344 dev_err(hdev->dev, "handle %u is mapped, cannot free\n",
346 spin_unlock(&vm->idr_lock);
351 * must remove from idr before the freeing of the physical
352 * pages as the refcount of the pool is also the trigger of the
355 idr_remove(&vm->phys_pg_pack_handles, handle);
356 spin_unlock(&vm->idr_lock);
358 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);
359 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
361 return free_phys_pg_pack(hdev, phys_pg_pack);
363 spin_unlock(&vm->idr_lock);
365 "free device memory failed, no match for handle %u\n",
374 * clear_va_list_locked() - free virtual addresses list.
375 * @hdev: habanalabs device structure.
376 * @va_list: list of virtual addresses to free.
378 * This function does the following:
379 * - Iterate over the list and free each virtual addresses block.
381 * This function should be called only when va_list lock is taken.
383 static void clear_va_list_locked(struct hl_device *hdev,
384 struct list_head *va_list)
386 struct hl_vm_va_block *va_block, *tmp;
388 list_for_each_entry_safe(va_block, tmp, va_list, node) {
389 list_del(&va_block->node);
395 * print_va_list_locked() - print virtual addresses list.
396 * @hdev: habanalabs device structure.
397 * @va_list: list of virtual addresses to print.
399 * This function does the following:
400 * - Iterate over the list and print each virtual addresses block.
402 * This function should be called only when va_list lock is taken.
404 static void print_va_list_locked(struct hl_device *hdev,
405 struct list_head *va_list)
408 struct hl_vm_va_block *va_block;
410 dev_dbg(hdev->dev, "print va list:\n");
412 list_for_each_entry(va_block, va_list, node)
414 "va block, start: 0x%llx, end: 0x%llx, size: %llu\n",
415 va_block->start, va_block->end, va_block->size);
420 * merge_va_blocks_locked() - merge a virtual block if possible.
421 * @hdev: pointer to the habanalabs device structure.
422 * @va_list: pointer to the virtual addresses block list.
423 * @va_block: virtual block to merge with adjacent blocks.
425 * This function does the following:
426 * - Merge the given blocks with the adjacent blocks if their virtual ranges
427 * create a contiguous virtual range.
429 * This Function should be called only when va_list lock is taken.
431 static void merge_va_blocks_locked(struct hl_device *hdev,
432 struct list_head *va_list, struct hl_vm_va_block *va_block)
434 struct hl_vm_va_block *prev, *next;
436 prev = list_prev_entry(va_block, node);
437 if (&prev->node != va_list && prev->end + 1 == va_block->start) {
438 prev->end = va_block->end;
439 prev->size = prev->end - prev->start;
440 list_del(&va_block->node);
445 next = list_next_entry(va_block, node);
446 if (&next->node != va_list && va_block->end + 1 == next->start) {
447 next->start = va_block->start;
448 next->size = next->end - next->start;
449 list_del(&va_block->node);
455 * add_va_block_locked() - add a virtual block to the virtual addresses list.
456 * @hdev: pointer to the habanalabs device structure.
457 * @va_list: pointer to the virtual addresses block list.
458 * @start: start virtual address.
459 * @end: end virtual address.
461 * This function does the following:
462 * - Add the given block to the virtual blocks list and merge with other blocks
463 * if a contiguous virtual block can be created.
465 * This Function should be called only when va_list lock is taken.
467 static int add_va_block_locked(struct hl_device *hdev,
468 struct list_head *va_list, u64 start, u64 end)
470 struct hl_vm_va_block *va_block, *res = NULL;
471 u64 size = end - start;
473 print_va_list_locked(hdev, va_list);
475 list_for_each_entry(va_block, va_list, node) {
476 /* TODO: remove upon matureness */
477 if (hl_mem_area_crosses_range(start, size, va_block->start,
480 "block crossing ranges at start 0x%llx, end 0x%llx\n",
481 va_block->start, va_block->end);
485 if (va_block->end < start)
489 va_block = kmalloc(sizeof(*va_block), GFP_KERNEL);
493 va_block->start = start;
495 va_block->size = size;
498 list_add(&va_block->node, va_list);
500 list_add(&va_block->node, &res->node);
502 merge_va_blocks_locked(hdev, va_list, va_block);
504 print_va_list_locked(hdev, va_list);
510 * add_va_block() - wrapper for add_va_block_locked.
511 * @hdev: pointer to the habanalabs device structure.
512 * @va_list: pointer to the virtual addresses block list.
513 * @start: start virtual address.
514 * @end: end virtual address.
516 * This function does the following:
517 * - Takes the list lock and calls add_va_block_locked.
519 static inline int add_va_block(struct hl_device *hdev,
520 struct hl_va_range *va_range, u64 start, u64 end)
524 mutex_lock(&va_range->lock);
525 rc = add_va_block_locked(hdev, &va_range->list, start, end);
526 mutex_unlock(&va_range->lock);
532 * is_hint_crossing_range() - check if hint address crossing specified reserved
535 static inline bool is_hint_crossing_range(enum hl_va_range_type range_type,
536 u64 start_addr, u32 size, struct asic_fixed_properties *prop) {
539 if (range_type == HL_VA_RANGE_TYPE_DRAM)
541 hl_mem_area_crosses_range(start_addr, size,
542 prop->hints_dram_reserved_va_range.start_addr,
543 prop->hints_dram_reserved_va_range.end_addr);
544 else if (range_type == HL_VA_RANGE_TYPE_HOST)
546 hl_mem_area_crosses_range(start_addr, size,
547 prop->hints_host_reserved_va_range.start_addr,
548 prop->hints_host_reserved_va_range.end_addr);
551 hl_mem_area_crosses_range(start_addr, size,
552 prop->hints_host_hpage_reserved_va_range.start_addr,
553 prop->hints_host_hpage_reserved_va_range.end_addr);
559 * get_va_block() - get a virtual block for the given size and alignment.
561 * @hdev: pointer to the habanalabs device structure.
562 * @va_range: pointer to the virtual addresses range.
563 * @size: requested block size.
564 * @hint_addr: hint for requested address by the user.
565 * @va_block_align: required alignment of the virtual block start address.
566 * @range_type: va range type (host, dram)
567 * @flags: additional memory flags, currently only uses HL_MEM_FORCE_HINT
569 * This function does the following:
570 * - Iterate on the virtual block list to find a suitable virtual block for the
571 * given size, hint address and alignment.
572 * - Reserve the requested block and update the list.
573 * - Return the start address of the virtual block.
575 static u64 get_va_block(struct hl_device *hdev,
576 struct hl_va_range *va_range,
577 u64 size, u64 hint_addr, u32 va_block_align,
578 enum hl_va_range_type range_type,
581 struct hl_vm_va_block *va_block, *new_va_block = NULL;
582 struct asic_fixed_properties *prop = &hdev->asic_prop;
583 u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end,
584 align_mask, reserved_valid_start = 0, reserved_valid_size = 0,
585 dram_hint_mask = prop->dram_hints_align_mask;
586 bool add_prev = false;
587 bool is_align_pow_2 = is_power_of_2(va_range->page_size);
588 bool is_hint_dram_addr = hl_is_dram_va(hdev, hint_addr);
589 bool force_hint = flags & HL_MEM_FORCE_HINT;
592 align_mask = ~((u64)va_block_align - 1);
595 * with non-power-of-2 range we work only with page granularity
596 * and the start address is page aligned,
597 * so no need for alignment checking.
599 size = DIV_ROUND_UP_ULL(size, va_range->page_size) *
602 tmp_hint_addr = hint_addr & ~dram_hint_mask;
604 /* Check if we need to ignore hint address */
605 if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) ||
606 (!is_align_pow_2 && is_hint_dram_addr &&
607 do_div(tmp_hint_addr, va_range->page_size))) {
610 /* Hint must be respected, so here we just fail */
612 "Hint address 0x%llx is not page aligned - cannot be respected\n",
618 "Hint address 0x%llx will be ignored because it is not aligned\n",
623 mutex_lock(&va_range->lock);
625 print_va_list_locked(hdev, &va_range->list);
627 list_for_each_entry(va_block, &va_range->list, node) {
628 /* Calc the first possible aligned addr */
629 valid_start = va_block->start;
631 if (is_align_pow_2 && (valid_start & (va_block_align - 1))) {
632 valid_start &= align_mask;
633 valid_start += va_block_align;
634 if (valid_start > va_block->end)
638 valid_size = va_block->end - valid_start;
639 if (valid_size < size)
643 * In case hint address is 0, and arc_hints_range_reservation
644 * property enabled, then avoid allocating va blocks from the
645 * range reserved for hint addresses
647 if (prop->hints_range_reservation && !hint_addr)
648 if (is_hint_crossing_range(range_type, valid_start,
652 /* Pick the minimal length block which has the required size */
653 if (!new_va_block || (valid_size < reserved_valid_size)) {
654 new_va_block = va_block;
655 reserved_valid_start = valid_start;
656 reserved_valid_size = valid_size;
659 if (hint_addr && hint_addr >= valid_start &&
660 (hint_addr + size) <= va_block->end) {
661 new_va_block = va_block;
662 reserved_valid_start = hint_addr;
663 reserved_valid_size = valid_size;
669 dev_err(hdev->dev, "no available va block for size %llu\n",
674 if (force_hint && reserved_valid_start != hint_addr) {
675 /* Hint address must be respected. If we are here - this means
676 * we could not respect it.
679 "Hint address 0x%llx could not be respected\n",
681 reserved_valid_start = 0;
686 * Check if there is some leftover range due to reserving the new
687 * va block, then return it to the main virtual addresses list.
689 if (reserved_valid_start > new_va_block->start) {
690 prev_start = new_va_block->start;
691 prev_end = reserved_valid_start - 1;
693 new_va_block->start = reserved_valid_start;
694 new_va_block->size = reserved_valid_size;
699 if (new_va_block->size > size) {
700 new_va_block->start += size;
701 new_va_block->size = new_va_block->end - new_va_block->start;
703 list_del(&new_va_block->node);
708 add_va_block_locked(hdev, &va_range->list, prev_start,
711 print_va_list_locked(hdev, &va_range->list);
713 mutex_unlock(&va_range->lock);
715 return reserved_valid_start;
719 * hl_reserve_va_block() - reserve a virtual block of a given size.
720 * @hdev: pointer to the habanalabs device structure.
721 * @ctx: current context
722 * @type: virtual addresses range type.
723 * @size: requested block size.
724 * @alignment: required alignment in bytes of the virtual block start address,
725 * 0 means no alignment.
727 * This function does the following:
728 * - Iterate on the virtual block list to find a suitable virtual block for the
729 * given size and alignment.
730 * - Reserve the requested block and update the list.
731 * - Return the start address of the virtual block.
733 u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
734 enum hl_va_range_type type, u32 size, u32 alignment)
736 return get_va_block(hdev, ctx->va_range[type], size, 0,
737 max(alignment, ctx->va_range[type]->page_size),
742 * hl_get_va_range_type() - get va_range type for the given address and size.
743 * @address: the start address of the area we want to validate.
744 * @size: the size in bytes of the area we want to validate.
745 * @type: returned va_range type.
747 * Return: true if the area is inside a valid range, false otherwise.
749 static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size,
750 enum hl_va_range_type *type)
754 for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX; i++) {
755 if (hl_mem_area_inside_range(address, size,
756 ctx->va_range[i]->start_addr,
757 ctx->va_range[i]->end_addr)) {
767 * hl_unreserve_va_block() - wrapper for add_va_block to unreserve a va block.
768 * @hdev: pointer to the habanalabs device structure
769 * @ctx: pointer to the context structure.
770 * @start: start virtual address.
771 * @end: end virtual address.
773 * This function does the following:
774 * - Takes the list lock and calls add_va_block_locked.
776 int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
777 u64 start_addr, u64 size)
779 enum hl_va_range_type type;
782 rc = hl_get_va_range_type(ctx, start_addr, size, &type);
785 "cannot find va_range for va %#llx size %llu",
790 rc = add_va_block(hdev, ctx->va_range[type], start_addr,
791 start_addr + size - 1);
794 "add va block failed for vaddr: 0x%llx\n", start_addr);
800 * init_phys_pg_pack_from_userptr() - initialize physical page pack from host
802 * @ctx: pointer to the context structure.
803 * @userptr: userptr to initialize from.
804 * @pphys_pg_pack: result pointer.
805 * @force_regular_page: tell the function to ignore huge page optimization,
806 * even if possible. Needed for cases where the device VA
807 * is allocated before we know the composition of the
810 * This function does the following:
811 * - Pin the physical pages related to the given virtual block.
812 * - Create a physical page pack from the physical pages related to the given
815 static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
816 struct hl_userptr *userptr,
817 struct hl_vm_phys_pg_pack **pphys_pg_pack,
818 bool force_regular_page)
820 u32 npages, page_size = PAGE_SIZE,
821 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
822 u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
823 struct hl_vm_phys_pg_pack *phys_pg_pack;
824 bool first = true, is_huge_page_opt;
825 u64 page_mask, total_npages;
826 struct scatterlist *sg;
830 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
834 phys_pg_pack->vm_type = userptr->vm_type;
835 phys_pg_pack->created_from_userptr = true;
836 phys_pg_pack->asid = ctx->asid;
837 atomic_set(&phys_pg_pack->mapping_cnt, 1);
839 is_huge_page_opt = (force_regular_page ? false : true);
841 /* Only if all dma_addrs are aligned to 2MB and their
842 * sizes is at least 2MB, we can use huge page mapping.
843 * We limit the 2MB optimization to this condition,
844 * since later on we acquire the related VA range as one
848 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
849 npages = hl_get_sg_info(sg, &dma_addr);
851 total_npages += npages;
853 if ((npages % pgs_in_huge_page) ||
854 (dma_addr & (huge_page_size - 1)))
855 is_huge_page_opt = false;
858 if (is_huge_page_opt) {
859 page_size = huge_page_size;
860 do_div(total_npages, pgs_in_huge_page);
863 page_mask = ~(((u64) page_size) - 1);
865 phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),
867 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
869 goto page_pack_arr_mem_err;
872 phys_pg_pack->npages = total_npages;
873 phys_pg_pack->page_size = page_size;
874 phys_pg_pack->total_size = total_npages * page_size;
877 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
878 npages = hl_get_sg_info(sg, &dma_addr);
880 /* align down to physical page size and save the offset */
883 phys_pg_pack->offset = dma_addr & (page_size - 1);
884 dma_addr &= page_mask;
888 phys_pg_pack->pages[j++] = dma_addr;
889 dma_addr += page_size;
891 if (is_huge_page_opt)
892 npages -= pgs_in_huge_page;
898 *pphys_pg_pack = phys_pg_pack;
902 page_pack_arr_mem_err:
909 * map_phys_pg_pack() - maps the physical page pack..
910 * @ctx: pointer to the context structure.
911 * @vaddr: start address of the virtual area to map from.
912 * @phys_pg_pack: the pack of physical pages to map to.
914 * This function does the following:
915 * - Maps each chunk of virtual memory to matching physical chunk.
916 * - Stores number of successful mappings in the given argument.
917 * - Returns 0 on success, error code otherwise.
919 static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
920 struct hl_vm_phys_pg_pack *phys_pg_pack)
922 struct hl_device *hdev = ctx->hdev;
923 u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;
924 u32 page_size = phys_pg_pack->page_size;
928 for (i = 0 ; i < phys_pg_pack->npages ; i++) {
929 paddr = phys_pg_pack->pages[i];
931 rc = hl_mmu_map_page(ctx, next_vaddr, paddr, page_size,
932 (i + 1) == phys_pg_pack->npages);
935 "map failed for handle %u, npages: %llu, mapped: %llu",
936 phys_pg_pack->handle, phys_pg_pack->npages,
942 next_vaddr += page_size;
948 is_host_addr = !hl_is_dram_va(hdev, vaddr);
951 for (i = 0 ; i < mapped_pg_cnt ; i++) {
952 if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
953 (i + 1) == mapped_pg_cnt))
954 dev_warn_ratelimited(hdev->dev,
955 "failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n",
956 phys_pg_pack->handle, next_vaddr,
957 phys_pg_pack->pages[i], page_size);
959 next_vaddr += page_size;
962 * unmapping on Palladium can be really long, so avoid a CPU
963 * soft lockup bug by sleeping a little between unmapping pages
965 * In addition, on host num of pages could be huge,
966 * because page size could be 4KB, so when unmapping host
967 * pages sleep every 32K pages to avoid soft lockup
969 if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0))
970 usleep_range(50, 200);
977 * unmap_phys_pg_pack() - unmaps the physical page pack.
978 * @ctx: pointer to the context structure.
979 * @vaddr: start address of the virtual area to unmap.
980 * @phys_pg_pack: the pack of physical pages to unmap.
982 static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
983 struct hl_vm_phys_pg_pack *phys_pg_pack)
985 struct hl_device *hdev = ctx->hdev;
990 is_host_addr = !hl_is_dram_va(hdev, vaddr);
991 page_size = phys_pg_pack->page_size;
994 for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {
995 if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
996 (i + 1) == phys_pg_pack->npages))
997 dev_warn_ratelimited(hdev->dev,
998 "unmap failed for vaddr: 0x%llx\n", next_vaddr);
1001 * unmapping on Palladium can be really long, so avoid a CPU
1002 * soft lockup bug by sleeping a little between unmapping pages
1004 * In addition, on host num of pages could be huge,
1005 * because page size could be 4KB, so when unmapping host
1006 * pages sleep every 32K pages to avoid soft lockup
1008 if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0))
1009 usleep_range(50, 200);
1013 static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args,
1016 struct hl_device *hdev = ctx->hdev;
1017 struct hl_vm *vm = &hdev->vm;
1018 struct hl_vm_phys_pg_pack *phys_pg_pack;
1021 handle = lower_32_bits(args->map_device.handle);
1022 spin_lock(&vm->idr_lock);
1023 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
1024 if (!phys_pg_pack) {
1025 spin_unlock(&vm->idr_lock);
1026 dev_err(hdev->dev, "no match for handle %u\n", handle);
1030 *paddr = phys_pg_pack->pages[0];
1032 spin_unlock(&vm->idr_lock);
1038 * map_device_va() - map the given memory.
1039 * @ctx: pointer to the context structure.
1040 * @args: host parameters with handle/host virtual address.
1041 * @device_addr: pointer to result device virtual address.
1043 * This function does the following:
1044 * - If given a physical device memory handle, map to a device virtual block
1045 * and return the start address of this block.
1046 * - If given a host virtual address and size, find the related physical pages,
1047 * map a device virtual block to this pages and return the start address of
1050 static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
1053 struct hl_device *hdev = ctx->hdev;
1054 struct hl_vm *vm = &hdev->vm;
1055 struct hl_vm_phys_pg_pack *phys_pg_pack;
1056 struct hl_userptr *userptr = NULL;
1057 struct hl_vm_hash_node *hnode;
1058 struct hl_va_range *va_range;
1059 enum vm_type *vm_type;
1060 u64 ret_vaddr, hint_addr;
1061 u32 handle = 0, va_block_align;
1063 bool is_userptr = args->flags & HL_MEM_USERPTR;
1064 enum hl_va_range_type va_range_type = 0;
1066 /* Assume failure */
1070 u64 addr = args->map_host.host_virt_addr,
1071 size = args->map_host.mem_size;
1072 u32 page_size = hdev->asic_prop.pmmu.page_size,
1073 huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
1075 rc = dma_map_host_va(hdev, addr, size, &userptr);
1077 dev_err(hdev->dev, "failed to get userptr from va\n");
1081 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1082 &phys_pg_pack, false);
1085 "unable to init page pack for vaddr 0x%llx\n",
1087 goto init_page_pack_err;
1090 vm_type = (enum vm_type *) userptr;
1091 hint_addr = args->map_host.hint_addr;
1092 handle = phys_pg_pack->handle;
1094 /* get required alignment */
1095 if (phys_pg_pack->page_size == page_size) {
1096 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1097 va_range_type = HL_VA_RANGE_TYPE_HOST;
1099 * huge page alignment may be needed in case of regular
1100 * page mapping, depending on the host VA alignment
1102 if (addr & (huge_page_size - 1))
1103 va_block_align = page_size;
1105 va_block_align = huge_page_size;
1108 * huge page alignment is needed in case of huge page
1111 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1112 va_range_type = HL_VA_RANGE_TYPE_HOST_HUGE;
1113 va_block_align = huge_page_size;
1116 handle = lower_32_bits(args->map_device.handle);
1118 spin_lock(&vm->idr_lock);
1119 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
1120 if (!phys_pg_pack) {
1121 spin_unlock(&vm->idr_lock);
1123 "no match for handle %u\n", handle);
1127 /* increment now to avoid freeing device memory while mapping */
1128 atomic_inc(&phys_pg_pack->mapping_cnt);
1130 spin_unlock(&vm->idr_lock);
1132 vm_type = (enum vm_type *) phys_pg_pack;
1134 hint_addr = args->map_device.hint_addr;
1136 /* DRAM VA alignment is the same as the MMU page size */
1137 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1138 va_range_type = HL_VA_RANGE_TYPE_DRAM;
1139 va_block_align = hdev->asic_prop.dmmu.page_size;
1143 * relevant for mapping device physical memory only, as host memory is
1146 if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&
1147 phys_pg_pack->asid != ctx->asid) {
1149 "Failed to map memory, handle %u is not shared\n",
1155 hnode = kzalloc(sizeof(*hnode), GFP_KERNEL);
1161 if (hint_addr && phys_pg_pack->offset) {
1162 if (args->flags & HL_MEM_FORCE_HINT) {
1163 /* Fail if hint must be respected but it can't be */
1165 "Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n",
1166 hint_addr, phys_pg_pack->offset);
1171 "Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n",
1172 hint_addr, phys_pg_pack->offset);
1175 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
1176 hint_addr, va_block_align,
1177 va_range_type, args->flags);
1179 dev_err(hdev->dev, "no available va block for handle %u\n",
1185 mutex_lock(&ctx->mmu_lock);
1187 rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
1189 mutex_unlock(&ctx->mmu_lock);
1190 dev_err(hdev->dev, "mapping page pack failed for handle %u\n",
1195 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, false,
1196 *vm_type, ctx->asid, ret_vaddr, phys_pg_pack->total_size);
1198 mutex_unlock(&ctx->mmu_lock);
1202 "mapping handle %u failed due to MMU cache invalidation\n",
1207 ret_vaddr += phys_pg_pack->offset;
1209 hnode->ptr = vm_type;
1210 hnode->vaddr = ret_vaddr;
1212 mutex_lock(&ctx->mem_hash_lock);
1213 hash_add(ctx->mem_hash, &hnode->node, ret_vaddr);
1214 mutex_unlock(&ctx->mem_hash_lock);
1216 *device_addr = ret_vaddr;
1219 rc = free_phys_pg_pack(hdev, phys_pg_pack);
1224 if (add_va_block(hdev, va_range, ret_vaddr,
1225 ret_vaddr + phys_pg_pack->total_size - 1))
1227 "release va block failed for handle 0x%x, vaddr: 0x%llx\n",
1234 atomic_dec(&phys_pg_pack->mapping_cnt);
1236 free_phys_pg_pack(hdev, phys_pg_pack);
1239 dma_unmap_host_va(hdev, userptr);
1245 * unmap_device_va() - unmap the given device virtual address.
1246 * @ctx: pointer to the context structure.
1247 * @args: host parameters with device virtual address to unmap.
1248 * @ctx_free: true if in context free flow, false otherwise.
1250 * This function does the following:
1251 * - unmap the physical pages related to the given virtual address.
1252 * - return the device virtual block to the virtual block list.
1254 static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
1257 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
1258 u64 vaddr = args->unmap.device_virt_addr;
1259 struct hl_vm_hash_node *hnode = NULL;
1260 struct asic_fixed_properties *prop;
1261 struct hl_device *hdev = ctx->hdev;
1262 struct hl_userptr *userptr = NULL;
1263 struct hl_va_range *va_range;
1264 enum vm_type *vm_type;
1268 prop = &hdev->asic_prop;
1270 /* protect from double entrance */
1271 mutex_lock(&ctx->mem_hash_lock);
1272 hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
1273 if (vaddr == hnode->vaddr)
1277 mutex_unlock(&ctx->mem_hash_lock);
1279 "unmap failed, no mem hnode for vaddr 0x%llx\n",
1284 hash_del(&hnode->node);
1285 mutex_unlock(&ctx->mem_hash_lock);
1287 vm_type = hnode->ptr;
1289 if (*vm_type == VM_TYPE_USERPTR) {
1291 userptr = hnode->ptr;
1293 rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack,
1297 "unable to init page pack for vaddr 0x%llx\n",
1302 if (phys_pg_pack->page_size ==
1303 hdev->asic_prop.pmmu.page_size)
1304 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1306 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1307 } else if (*vm_type == VM_TYPE_PHYS_PACK) {
1309 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1310 phys_pg_pack = hnode->ptr;
1313 "unmap failed, unknown vm desc for vaddr 0x%llx\n",
1319 if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {
1320 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr);
1322 goto mapping_cnt_err;
1325 if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size))
1326 vaddr = prop->dram_base_address +
1327 DIV_ROUND_DOWN_ULL(vaddr - prop->dram_base_address,
1328 phys_pg_pack->page_size) *
1329 phys_pg_pack->page_size;
1331 vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);
1333 mutex_lock(&ctx->mmu_lock);
1335 unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);
1338 * During context free this function is called in a loop to clean all
1339 * the context mappings. Hence the cache invalidation can be called once
1340 * at the loop end rather than for each iteration
1343 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, true,
1344 *vm_type, ctx->asid, vaddr,
1345 phys_pg_pack->total_size);
1347 mutex_unlock(&ctx->mmu_lock);
1350 * If the context is closing we don't need to check for the MMU cache
1351 * invalidation return code and update the VA free list as in this flow
1352 * we invalidate the MMU cache outside of this unmap function and the VA
1353 * free list will be freed anyway.
1360 "unmapping vaddr 0x%llx failed due to MMU cache invalidation\n",
1363 tmp_rc = add_va_block(hdev, va_range, vaddr,
1364 vaddr + phys_pg_pack->total_size - 1);
1367 "add va block failed for vaddr: 0x%llx\n",
1374 atomic_dec(&phys_pg_pack->mapping_cnt);
1378 free_phys_pg_pack(hdev, phys_pg_pack);
1379 dma_unmap_host_va(hdev, userptr);
1386 free_phys_pg_pack(hdev, phys_pg_pack);
1388 mutex_lock(&ctx->mem_hash_lock);
1389 hash_add(ctx->mem_hash, &hnode->node, vaddr);
1390 mutex_unlock(&ctx->mem_hash_lock);
1395 static int map_block(struct hl_device *hdev, u64 address, u64 *handle,
1401 rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id);
1403 *handle = block_id | HL_MMAP_TYPE_BLOCK;
1404 *handle <<= PAGE_SHIFT;
1409 static void hw_block_vm_close(struct vm_area_struct *vma)
1411 struct hl_vm_hw_block_list_node *lnode =
1412 (struct hl_vm_hw_block_list_node *) vma->vm_private_data;
1413 struct hl_ctx *ctx = lnode->ctx;
1415 mutex_lock(&ctx->hw_block_list_lock);
1416 list_del(&lnode->node);
1417 mutex_unlock(&ctx->hw_block_list_lock);
1420 vma->vm_private_data = NULL;
1423 static const struct vm_operations_struct hw_block_vm_ops = {
1424 .close = hw_block_vm_close
1428 * hl_hw_block_mmap() - mmap a hw block to user.
1429 * @hpriv: pointer to the private data of the fd
1430 * @vma: pointer to vm_area_struct of the process
1432 * Driver increments context reference for every HW block mapped in order
1433 * to prevent user from closing FD without unmapping first
1435 int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
1437 struct hl_vm_hw_block_list_node *lnode;
1438 struct hl_device *hdev = hpriv->hdev;
1439 struct hl_ctx *ctx = hpriv->ctx;
1440 u32 block_id, block_size;
1443 /* We use the page offset to hold the block id and thus we need to clear
1444 * it before doing the mmap itself
1446 block_id = vma->vm_pgoff;
1449 /* Driver only allows mapping of a complete HW block */
1450 block_size = vma->vm_end - vma->vm_start;
1452 if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) {
1454 "user pointer is invalid - 0x%lx\n",
1460 lnode = kzalloc(sizeof(*lnode), GFP_KERNEL);
1464 vma->vm_ops = &hw_block_vm_ops;
1465 vma->vm_private_data = lnode;
1467 hl_ctx_get(hdev, ctx);
1469 rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size);
1477 lnode->vaddr = vma->vm_start;
1478 lnode->size = block_size;
1479 lnode->id = block_id;
1481 mutex_lock(&ctx->hw_block_list_lock);
1482 list_add_tail(&lnode->node, &ctx->hw_block_mem_list);
1483 mutex_unlock(&ctx->hw_block_list_lock);
1485 vma->vm_pgoff = block_id;
1490 static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args)
1492 struct hl_device *hdev = hpriv->hdev;
1493 struct hl_ctx *ctx = hpriv->ctx;
1494 u64 block_handle, device_addr = 0;
1495 u32 handle = 0, block_size;
1498 switch (args->in.op) {
1499 case HL_MEM_OP_ALLOC:
1500 if (args->in.alloc.mem_size == 0) {
1502 "alloc size must be larger than 0\n");
1507 /* Force contiguous as there are no real MMU
1508 * translations to overcome physical memory gaps
1510 args->in.flags |= HL_MEM_CONTIGUOUS;
1511 rc = alloc_device_memory(ctx, &args->in, &handle);
1513 memset(args, 0, sizeof(*args));
1514 args->out.handle = (__u64) handle;
1517 case HL_MEM_OP_FREE:
1518 rc = free_device_memory(ctx, &args->in);
1522 if (args->in.flags & HL_MEM_USERPTR) {
1523 device_addr = args->in.map_host.host_virt_addr;
1526 rc = get_paddr_from_handle(ctx, &args->in,
1530 memset(args, 0, sizeof(*args));
1531 args->out.device_virt_addr = device_addr;
1534 case HL_MEM_OP_UNMAP:
1538 case HL_MEM_OP_MAP_BLOCK:
1539 rc = map_block(hdev, args->in.map_block.block_addr,
1540 &block_handle, &block_size);
1541 args->out.block_handle = block_handle;
1542 args->out.block_size = block_size;
1546 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1555 int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
1557 enum hl_device_status status;
1558 union hl_mem_args *args = data;
1559 struct hl_device *hdev = hpriv->hdev;
1560 struct hl_ctx *ctx = hpriv->ctx;
1561 u64 block_handle, device_addr = 0;
1562 u32 handle = 0, block_size;
1565 if (!hl_device_operational(hdev, &status)) {
1566 dev_warn_ratelimited(hdev->dev,
1567 "Device is %s. Can't execute MEMORY IOCTL\n",
1568 hdev->status[status]);
1572 if (!hdev->mmu_enable)
1573 return mem_ioctl_no_mmu(hpriv, args);
1575 switch (args->in.op) {
1576 case HL_MEM_OP_ALLOC:
1577 if (args->in.alloc.mem_size == 0) {
1579 "alloc size must be larger than 0\n");
1584 /* If DRAM does not support virtual memory the driver won't
1585 * handle the allocation/freeing of that memory. However, for
1586 * system administration/monitoring purposes, the driver will
1587 * keep track of the amount of DRAM memory that is allocated
1588 * and freed by the user. Because this code totally relies on
1589 * the user's input, the driver can't ensure the validity
1590 * of this accounting.
1592 if (!hdev->asic_prop.dram_supports_virtual_memory) {
1593 atomic64_add(args->in.alloc.mem_size,
1594 &ctx->dram_phys_mem);
1595 atomic64_add(args->in.alloc.mem_size,
1596 &hdev->dram_used_mem);
1598 dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
1601 memset(args, 0, sizeof(*args));
1602 args->out.handle = 0;
1606 rc = alloc_device_memory(ctx, &args->in, &handle);
1608 memset(args, 0, sizeof(*args));
1609 args->out.handle = (__u64) handle;
1612 case HL_MEM_OP_FREE:
1613 /* If DRAM does not support virtual memory the driver won't
1614 * handle the allocation/freeing of that memory. However, for
1615 * system administration/monitoring purposes, the driver will
1616 * keep track of the amount of DRAM memory that is allocated
1617 * and freed by the user. Because this code totally relies on
1618 * the user's input, the driver can't ensure the validity
1619 * of this accounting.
1621 if (!hdev->asic_prop.dram_supports_virtual_memory) {
1622 atomic64_sub(args->in.alloc.mem_size,
1623 &ctx->dram_phys_mem);
1624 atomic64_sub(args->in.alloc.mem_size,
1625 &hdev->dram_used_mem);
1627 dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
1633 rc = free_device_memory(ctx, &args->in);
1637 rc = map_device_va(ctx, &args->in, &device_addr);
1639 memset(args, 0, sizeof(*args));
1640 args->out.device_virt_addr = device_addr;
1643 case HL_MEM_OP_UNMAP:
1644 rc = unmap_device_va(ctx, &args->in, false);
1647 case HL_MEM_OP_MAP_BLOCK:
1648 rc = map_block(hdev, args->in.map_block.block_addr,
1649 &block_handle, &block_size);
1650 args->out.block_handle = block_handle;
1651 args->out.block_size = block_size;
1655 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1664 static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
1665 u32 npages, u64 start, u32 offset,
1666 struct hl_userptr *userptr)
1670 if (!access_ok((void __user *) (uintptr_t) addr, size)) {
1671 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr);
1675 userptr->pages = kvmalloc_array(npages, sizeof(*userptr->pages),
1677 if (!userptr->pages)
1680 rc = pin_user_pages_fast(start, npages,
1681 FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM,
1686 "Failed (%d) to pin host memory with user ptr 0x%llx, size 0x%llx, npages %d\n",
1687 rc, addr, size, npages);
1694 userptr->npages = npages;
1696 rc = sg_alloc_table_from_pages(userptr->sgt,
1698 npages, offset, size, GFP_KERNEL);
1700 dev_err(hdev->dev, "failed to create SG table from pages\n");
1707 unpin_user_pages(userptr->pages, npages);
1709 kvfree(userptr->pages);
1714 * hl_pin_host_memory() - pins a chunk of host memory.
1715 * @hdev: pointer to the habanalabs device structure.
1716 * @addr: the host virtual address of the memory area.
1717 * @size: the size of the memory area.
1718 * @userptr: pointer to hl_userptr structure.
1720 * This function does the following:
1721 * - Pins the physical pages.
1722 * - Create an SG list from those pages.
1724 int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1725 struct hl_userptr *userptr)
1732 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size);
1737 * If the combination of the address and size requested for this memory
1738 * region causes an integer overflow, return error.
1740 if (((addr + size) < addr) ||
1741 PAGE_ALIGN(addr + size) < (addr + size)) {
1743 "user pointer 0x%llx + %llu causes integer overflow\n",
1748 userptr->pid = current->pid;
1749 userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL);
1753 start = addr & PAGE_MASK;
1754 offset = addr & ~PAGE_MASK;
1755 end = PAGE_ALIGN(addr + size);
1756 npages = (end - start) >> PAGE_SHIFT;
1758 userptr->size = size;
1759 userptr->addr = addr;
1760 userptr->dma_mapped = false;
1761 INIT_LIST_HEAD(&userptr->job_node);
1763 rc = get_user_memory(hdev, addr, size, npages, start, offset,
1767 "failed to get user memory for address 0x%llx\n",
1772 hl_debugfs_add_userptr(hdev, userptr);
1777 kfree(userptr->sgt);
1782 * hl_unpin_host_memory - unpins a chunk of host memory.
1783 * @hdev: pointer to the habanalabs device structure
1784 * @userptr: pointer to hl_userptr structure
1786 * This function does the following:
1787 * - Unpins the physical pages related to the host memory
1788 * - Free the SG list
1790 void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
1792 hl_debugfs_remove_userptr(hdev, userptr);
1794 if (userptr->dma_mapped)
1795 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl,
1796 userptr->sgt->nents,
1799 unpin_user_pages_dirty_lock(userptr->pages, userptr->npages, true);
1800 kvfree(userptr->pages);
1802 list_del(&userptr->job_node);
1804 sg_free_table(userptr->sgt);
1805 kfree(userptr->sgt);
1809 * hl_userptr_delete_list() - clear userptr list.
1810 * @hdev: pointer to the habanalabs device structure.
1811 * @userptr_list: pointer to the list to clear.
1813 * This function does the following:
1814 * - Iterates over the list and unpins the host memory and frees the userptr
1817 void hl_userptr_delete_list(struct hl_device *hdev,
1818 struct list_head *userptr_list)
1820 struct hl_userptr *userptr, *tmp;
1822 list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) {
1823 hl_unpin_host_memory(hdev, userptr);
1827 INIT_LIST_HEAD(userptr_list);
1831 * hl_userptr_is_pinned() - returns whether the given userptr is pinned.
1832 * @hdev: pointer to the habanalabs device structure.
1833 * @userptr_list: pointer to the list to clear.
1834 * @userptr: pointer to userptr to check.
1836 * This function does the following:
1837 * - Iterates over the list and checks if the given userptr is in it, means is
1838 * pinned. If so, returns true, otherwise returns false.
1840 bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
1841 u32 size, struct list_head *userptr_list,
1842 struct hl_userptr **userptr)
1844 list_for_each_entry((*userptr), userptr_list, job_node) {
1845 if ((addr == (*userptr)->addr) && (size == (*userptr)->size))
1853 * va_range_init() - initialize virtual addresses range.
1854 * @hdev: pointer to the habanalabs device structure.
1855 * @va_range: pointer to the range to initialize.
1856 * @start: range start address.
1857 * @end: range end address.
1859 * This function does the following:
1860 * - Initializes the virtual addresses list of the given range with the given
1863 static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
1864 u64 start, u64 end, u32 page_size)
1868 INIT_LIST_HEAD(&va_range->list);
1871 * PAGE_SIZE alignment
1872 * it is the callers responsibility to align the addresses if the
1873 * page size is not a power of 2
1876 if (is_power_of_2(page_size)) {
1877 if (start & (PAGE_SIZE - 1)) {
1882 if (end & (PAGE_SIZE - 1))
1887 dev_err(hdev->dev, "too small vm range for va list\n");
1891 rc = add_va_block(hdev, va_range, start, end);
1894 dev_err(hdev->dev, "Failed to init host va list\n");
1898 va_range->start_addr = start;
1899 va_range->end_addr = end;
1900 va_range->page_size = page_size;
1906 * va_range_fini() - clear a virtual addresses range.
1907 * @hdev: pointer to the habanalabs structure.
1908 * va_range: pointer to virtual addresses rang.e
1910 * This function does the following:
1911 * - Frees the virtual addresses block list and its lock.
1913 static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range)
1915 mutex_lock(&va_range->lock);
1916 clear_va_list_locked(hdev, &va_range->list);
1917 mutex_unlock(&va_range->lock);
1919 mutex_destroy(&va_range->lock);
1924 * vm_ctx_init_with_ranges() - initialize virtual memory for context.
1925 * @ctx: pointer to the habanalabs context structure.
1926 * @host_range_start: host virtual addresses range start.
1927 * @host_range_end: host virtual addresses range end.
1928 * @host_huge_range_start: host virtual addresses range start for memory
1929 * allocated with huge pages.
1930 * @host_huge_range_end: host virtual addresses range end for memory allocated
1932 * @dram_range_start: dram virtual addresses range start.
1933 * @dram_range_end: dram virtual addresses range end.
1935 * This function initializes the following:
1936 * - MMU for context.
1937 * - Virtual address to area descriptor hashtable.
1938 * - Virtual block list of available virtual memory.
1940 static int vm_ctx_init_with_ranges(struct hl_ctx *ctx,
1941 u64 host_range_start,
1944 u64 host_huge_range_start,
1945 u64 host_huge_range_end,
1946 u32 host_huge_page_size,
1947 u64 dram_range_start,
1951 struct hl_device *hdev = ctx->hdev;
1954 for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) {
1956 kzalloc(sizeof(struct hl_va_range), GFP_KERNEL);
1957 if (!ctx->va_range[i]) {
1963 rc = hl_mmu_ctx_init(ctx);
1965 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
1969 mutex_init(&ctx->mem_hash_lock);
1970 hash_init(ctx->mem_hash);
1972 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1974 rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST],
1975 host_range_start, host_range_end, host_page_size);
1977 dev_err(hdev->dev, "failed to init host vm range\n");
1981 if (hdev->pmmu_huge_range) {
1982 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1984 rc = va_range_init(hdev,
1985 ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE],
1986 host_huge_range_start, host_huge_range_end,
1987 host_huge_page_size);
1990 "failed to init host huge vm range\n");
1991 goto clear_host_va_range;
1994 kfree(ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
1995 ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] =
1996 ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1999 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
2001 rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM],
2002 dram_range_start, dram_range_end, dram_page_size);
2004 dev_err(hdev->dev, "failed to init dram vm range\n");
2005 goto clear_host_huge_va_range;
2008 hl_debugfs_add_ctx_mem_hash(hdev, ctx);
2012 clear_host_huge_va_range:
2013 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
2015 if (hdev->pmmu_huge_range) {
2016 mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2017 clear_va_list_locked(hdev,
2018 &ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list);
2019 mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2021 clear_host_va_range:
2022 if (hdev->pmmu_huge_range)
2023 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
2024 mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2025 clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list);
2026 mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2028 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
2029 mutex_destroy(&ctx->mem_hash_lock);
2030 hl_mmu_ctx_fini(ctx);
2032 for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++)
2033 kfree(ctx->va_range[i]);
2038 int hl_vm_ctx_init(struct hl_ctx *ctx)
2040 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
2041 u64 host_range_start, host_range_end, host_huge_range_start,
2042 host_huge_range_end, dram_range_start, dram_range_end;
2043 u32 host_page_size, host_huge_page_size, dram_page_size;
2045 atomic64_set(&ctx->dram_phys_mem, 0);
2048 * - If MMU is enabled, init the ranges as usual.
2049 * - If MMU is disabled, in case of host mapping, the returned address
2051 * In case of DRAM mapping, the returned address is the physical
2052 * address of the memory related to the given handle.
2054 if (!ctx->hdev->mmu_enable)
2057 dram_range_start = prop->dmmu.start_addr;
2058 dram_range_end = prop->dmmu.end_addr;
2059 dram_page_size = prop->dram_page_size ?
2060 prop->dram_page_size : prop->dmmu.page_size;
2061 host_range_start = prop->pmmu.start_addr;
2062 host_range_end = prop->pmmu.end_addr;
2063 host_page_size = prop->pmmu.page_size;
2064 host_huge_range_start = prop->pmmu_huge.start_addr;
2065 host_huge_range_end = prop->pmmu_huge.end_addr;
2066 host_huge_page_size = prop->pmmu_huge.page_size;
2068 return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end,
2069 host_page_size, host_huge_range_start,
2070 host_huge_range_end, host_huge_page_size,
2071 dram_range_start, dram_range_end, dram_page_size);
2075 * hl_vm_ctx_fini() - virtual memory teardown of context.
2076 * @ctx: pointer to the habanalabs context structure.
2078 * This function perform teardown the following:
2079 * - Virtual block list of available virtual memory.
2080 * - Virtual address to area descriptor hashtable.
2081 * - MMU for context.
2083 * In addition this function does the following:
2084 * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
2085 * hashtable should be empty as no valid mappings should exist at this
2087 * - Frees any existing physical page list from the idr which relates to the
2088 * current context asid.
2089 * - This function checks the virtual block list for correctness. At this point
2090 * the list should contain one element which describes the whole virtual
2091 * memory range of the context. Otherwise, a warning is printed.
2093 void hl_vm_ctx_fini(struct hl_ctx *ctx)
2095 struct hl_device *hdev = ctx->hdev;
2096 struct hl_vm *vm = &hdev->vm;
2097 struct hl_vm_phys_pg_pack *phys_pg_list;
2098 struct hl_vm_hash_node *hnode;
2099 struct hlist_node *tmp_node;
2100 struct hl_mem_in args;
2103 if (!hdev->mmu_enable)
2106 hl_debugfs_remove_ctx_mem_hash(hdev, ctx);
2109 * Clearly something went wrong on hard reset so no point in printing
2110 * another side effect error
2112 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
2114 "user released device without removing its memory mappings\n");
2116 hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {
2118 "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n",
2119 hnode->vaddr, ctx->asid);
2120 args.unmap.device_virt_addr = hnode->vaddr;
2121 unmap_device_va(ctx, &args, true);
2124 mutex_lock(&ctx->mmu_lock);
2126 /* invalidate the cache once after the unmapping loop */
2127 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
2128 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK);
2130 mutex_unlock(&ctx->mmu_lock);
2132 spin_lock(&vm->idr_lock);
2133 idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i)
2134 if (phys_pg_list->asid == ctx->asid) {
2136 "page list 0x%px of asid %d is still alive\n",
2137 phys_pg_list, ctx->asid);
2138 atomic64_sub(phys_pg_list->total_size,
2139 &hdev->dram_used_mem);
2140 free_phys_pg_pack(hdev, phys_pg_list);
2141 idr_remove(&vm->phys_pg_pack_handles, i);
2143 spin_unlock(&vm->idr_lock);
2145 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]);
2146 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]);
2148 if (hdev->pmmu_huge_range)
2149 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
2151 mutex_destroy(&ctx->mem_hash_lock);
2152 hl_mmu_ctx_fini(ctx);
2154 /* In this case we need to clear the global accounting of DRAM usage
2155 * because the user notifies us on allocations. If the user is no more,
2156 * all DRAM is available
2158 if (ctx->asid != HL_KERNEL_ASID_ID &&
2159 !hdev->asic_prop.dram_supports_virtual_memory)
2160 atomic64_set(&hdev->dram_used_mem, 0);
2164 * hl_vm_init() - initialize virtual memory module.
2165 * @hdev: pointer to the habanalabs device structure.
2167 * This function initializes the following:
2169 * - DRAM physical pages pool of 2MB.
2170 * - Idr for device memory allocation handles.
2172 int hl_vm_init(struct hl_device *hdev)
2174 struct asic_fixed_properties *prop = &hdev->asic_prop;
2175 struct hl_vm *vm = &hdev->vm;
2178 if (is_power_of_2(prop->dram_page_size))
2180 gen_pool_create(__ffs(prop->dram_page_size), -1);
2183 gen_pool_create(__ffs(DRAM_POOL_PAGE_SIZE), -1);
2185 if (!vm->dram_pg_pool) {
2186 dev_err(hdev->dev, "Failed to create dram page pool\n");
2190 kref_init(&vm->dram_pg_pool_refcount);
2192 rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address,
2193 prop->dram_end_address - prop->dram_user_base_address,
2198 "Failed to add memory to dram page pool %d\n", rc);
2202 spin_lock_init(&vm->idr_lock);
2203 idr_init(&vm->phys_pg_pack_handles);
2205 atomic64_set(&hdev->dram_used_mem, 0);
2207 vm->init_done = true;
2212 gen_pool_destroy(vm->dram_pg_pool);
2218 * hl_vm_fini() - virtual memory module teardown.
2219 * @hdev: pointer to the habanalabs device structure.
2221 * This function perform teardown to the following:
2222 * - Idr for device memory allocation handles.
2223 * - DRAM physical pages pool of 2MB.
2226 void hl_vm_fini(struct hl_device *hdev)
2228 struct hl_vm *vm = &hdev->vm;
2234 * At this point all the contexts should be freed and hence no DRAM
2235 * memory should be in use. Hence the DRAM pool should be freed here.
2237 if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1)
2238 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n",
2241 vm->init_done = false;
2245 * hl_hw_block_mem_init() - HW block memory initialization.
2246 * @ctx: pointer to the habanalabs context structure.
2248 * This function initializes the HW block virtual mapped addresses list and
2251 void hl_hw_block_mem_init(struct hl_ctx *ctx)
2253 mutex_init(&ctx->hw_block_list_lock);
2254 INIT_LIST_HEAD(&ctx->hw_block_mem_list);
2258 * hl_hw_block_mem_fini() - HW block memory teardown.
2259 * @ctx: pointer to the habanalabs context structure.
2261 * This function clears the HW block virtual mapped addresses list and destroys
2264 void hl_hw_block_mem_fini(struct hl_ctx *ctx)
2266 struct hl_vm_hw_block_list_node *lnode, *tmp;
2268 if (!list_empty(&ctx->hw_block_mem_list))
2269 dev_crit(ctx->hdev->dev, "HW block mem list isn't empty\n");
2271 list_for_each_entry_safe(lnode, tmp, &ctx->hw_block_mem_list, node) {
2272 list_del(&lnode->node);
2276 mutex_destroy(&ctx->hw_block_list_lock);