Merge branch 'mhi-net-immutable' of https://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-microblaze.git] / drivers / misc / habanalabs / common / memory.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4  * Copyright 2016-2019 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
10 #include "../include/hw_ip/mmu/mmu_general.h"
11
12 #include <linux/uaccess.h>
13 #include <linux/slab.h>
14
15 #define HL_MMU_DEBUG    0
16
17 /*
18  * The va ranges in context object contain a list with the available chunks of
19  * device virtual memory.
20  * There is one range for host allocations and one for DRAM allocations.
21  *
22  * On initialization each range contains one chunk of all of its available
23  * virtual range which is a half of the total device virtual range.
24  *
25  * On each mapping of physical pages, a suitable virtual range chunk (with a
26  * minimum size) is selected from the list. If the chunk size equals the
27  * requested size, the chunk is returned. Otherwise, the chunk is split into
28  * two chunks - one to return as result and a remainder to stay in the list.
29  *
30  * On each Unmapping of a virtual address, the relevant virtual chunk is
31  * returned to the list. The chunk is added to the list and if its edges match
32  * the edges of the adjacent chunks (means a contiguous chunk can be created),
33  * the chunks are merged.
34  *
35  * On finish, the list is checked to have only one chunk of all the relevant
36  * virtual range (which is a half of the device total virtual range).
37  * If not (means not all mappings were unmapped), a warning is printed.
38  */
39
40 /*
41  * alloc_device_memory - allocate device memory
42  *
43  * @ctx                 : current context
44  * @args                : host parameters containing the requested size
45  * @ret_handle          : result handle
46  *
47  * This function does the following:
48  * - Allocate the requested size rounded up to 'dram_page_size' pages
49  * - Return unique handle
50  */
51 static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
52                                 u32 *ret_handle)
53 {
54         struct hl_device *hdev = ctx->hdev;
55         struct hl_vm *vm = &hdev->vm;
56         struct hl_vm_phys_pg_pack *phys_pg_pack;
57         u64 paddr = 0, total_size, num_pgs, i;
58         u32 num_curr_pgs, page_size, page_shift;
59         int handle, rc;
60         bool contiguous;
61
62         num_curr_pgs = 0;
63         page_size = hdev->asic_prop.dram_page_size;
64         page_shift = __ffs(page_size);
65         num_pgs = (args->alloc.mem_size + (page_size - 1)) >> page_shift;
66         total_size = num_pgs << page_shift;
67
68         if (!total_size) {
69                 dev_err(hdev->dev, "Cannot allocate 0 bytes\n");
70                 return -EINVAL;
71         }
72
73         contiguous = args->flags & HL_MEM_CONTIGUOUS;
74
75         if (contiguous) {
76                 paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size);
77                 if (!paddr) {
78                         dev_err(hdev->dev,
79                                 "failed to allocate %llu contiguous pages with total size of %llu\n",
80                                 num_pgs, total_size);
81                         return -ENOMEM;
82                 }
83
84                 if (hdev->memory_scrub) {
85                         rc = hdev->asic_funcs->scrub_device_mem(hdev, paddr,
86                                         total_size);
87                         if (rc) {
88                                 dev_err(hdev->dev,
89                                         "Failed to scrub contiguous device memory\n");
90                                 goto pages_pack_err;
91                         }
92                 }
93         }
94
95         phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
96         if (!phys_pg_pack) {
97                 rc = -ENOMEM;
98                 goto pages_pack_err;
99         }
100
101         phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;
102         phys_pg_pack->asid = ctx->asid;
103         phys_pg_pack->npages = num_pgs;
104         phys_pg_pack->page_size = page_size;
105         phys_pg_pack->total_size = total_size;
106         phys_pg_pack->flags = args->flags;
107         phys_pg_pack->contiguous = contiguous;
108
109         phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);
110         if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
111                 rc = -ENOMEM;
112                 goto pages_arr_err;
113         }
114
115         if (phys_pg_pack->contiguous) {
116                 for (i = 0 ; i < num_pgs ; i++)
117                         phys_pg_pack->pages[i] = paddr + i * page_size;
118         } else {
119                 for (i = 0 ; i < num_pgs ; i++) {
120                         phys_pg_pack->pages[i] = (u64) gen_pool_alloc(
121                                                         vm->dram_pg_pool,
122                                                         page_size);
123                         if (!phys_pg_pack->pages[i]) {
124                                 dev_err(hdev->dev,
125                                         "Failed to allocate device memory (out of memory)\n");
126                                 rc = -ENOMEM;
127                                 goto page_err;
128                         }
129
130                         if (hdev->memory_scrub) {
131                                 rc = hdev->asic_funcs->scrub_device_mem(hdev,
132                                                 phys_pg_pack->pages[i],
133                                                 page_size);
134                                 if (rc) {
135                                         dev_err(hdev->dev,
136                                                 "Failed to scrub device memory\n");
137                                         goto page_err;
138                                 }
139                         }
140
141                         num_curr_pgs++;
142                 }
143         }
144
145         spin_lock(&vm->idr_lock);
146         handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,
147                                 GFP_ATOMIC);
148         spin_unlock(&vm->idr_lock);
149
150         if (handle < 0) {
151                 dev_err(hdev->dev, "Failed to get handle for page\n");
152                 rc = -EFAULT;
153                 goto idr_err;
154         }
155
156         for (i = 0 ; i < num_pgs ; i++)
157                 kref_get(&vm->dram_pg_pool_refcount);
158
159         phys_pg_pack->handle = handle;
160
161         atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);
162         atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);
163
164         *ret_handle = handle;
165
166         return 0;
167
168 idr_err:
169 page_err:
170         if (!phys_pg_pack->contiguous)
171                 for (i = 0 ; i < num_curr_pgs ; i++)
172                         gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],
173                                         page_size);
174
175         kvfree(phys_pg_pack->pages);
176 pages_arr_err:
177         kfree(phys_pg_pack);
178 pages_pack_err:
179         if (contiguous)
180                 gen_pool_free(vm->dram_pg_pool, paddr, total_size);
181
182         return rc;
183 }
184
185 /*
186  * dma_map_host_va - DMA mapping of the given host virtual address.
187  * @hdev: habanalabs device structure
188  * @addr: the host virtual address of the memory area
189  * @size: the size of the memory area
190  * @p_userptr: pointer to result userptr structure
191  *
192  * This function does the following:
193  * - Allocate userptr structure
194  * - Pin the given host memory using the userptr structure
195  * - Perform DMA mapping to have the DMA addresses of the pages
196  */
197 static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
198                                 struct hl_userptr **p_userptr)
199 {
200         struct hl_userptr *userptr;
201         int rc;
202
203         userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
204         if (!userptr) {
205                 rc = -ENOMEM;
206                 goto userptr_err;
207         }
208
209         rc = hl_pin_host_memory(hdev, addr, size, userptr);
210         if (rc) {
211                 dev_err(hdev->dev, "Failed to pin host memory\n");
212                 goto pin_err;
213         }
214
215         rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
216                                         userptr->sgt->nents, DMA_BIDIRECTIONAL);
217         if (rc) {
218                 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
219                 goto dma_map_err;
220         }
221
222         userptr->dma_mapped = true;
223         userptr->dir = DMA_BIDIRECTIONAL;
224         userptr->vm_type = VM_TYPE_USERPTR;
225
226         *p_userptr = userptr;
227
228         return 0;
229
230 dma_map_err:
231         hl_unpin_host_memory(hdev, userptr);
232 pin_err:
233         kfree(userptr);
234 userptr_err:
235
236         return rc;
237 }
238
239 /*
240  * dma_unmap_host_va - DMA unmapping of the given host virtual address.
241  * @hdev: habanalabs device structure
242  * @userptr: userptr to free
243  *
244  * This function does the following:
245  * - Unpins the physical pages
246  * - Frees the userptr structure
247  */
248 static void dma_unmap_host_va(struct hl_device *hdev,
249                                 struct hl_userptr *userptr)
250 {
251         hl_unpin_host_memory(hdev, userptr);
252         kfree(userptr);
253 }
254
255 /*
256  * dram_pg_pool_do_release - free DRAM pages pool
257  *
258  * @ref                 : pointer to reference object
259  *
260  * This function does the following:
261  * - Frees the idr structure of physical pages handles
262  * - Frees the generic pool of DRAM physical pages
263  */
264 static void dram_pg_pool_do_release(struct kref *ref)
265 {
266         struct hl_vm *vm = container_of(ref, struct hl_vm,
267                         dram_pg_pool_refcount);
268
269         /*
270          * free the idr here as only here we know for sure that there are no
271          * allocated physical pages and hence there are no handles in use
272          */
273         idr_destroy(&vm->phys_pg_pack_handles);
274         gen_pool_destroy(vm->dram_pg_pool);
275 }
276
277 /*
278  * free_phys_pg_pack - free physical page pack
279  * @hdev: habanalabs device structure
280  * @phys_pg_pack: physical page pack to free
281  *
282  * This function does the following:
283  * - For DRAM memory only, iterate over the pack and free each physical block
284  *   structure by returning it to the general pool
285  * - Free the hl_vm_phys_pg_pack structure
286  */
287 static void free_phys_pg_pack(struct hl_device *hdev,
288                                 struct hl_vm_phys_pg_pack *phys_pg_pack)
289 {
290         struct hl_vm *vm = &hdev->vm;
291         u64 i;
292
293         if (!phys_pg_pack->created_from_userptr) {
294                 if (phys_pg_pack->contiguous) {
295                         gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],
296                                         phys_pg_pack->total_size);
297
298                         for (i = 0; i < phys_pg_pack->npages ; i++)
299                                 kref_put(&vm->dram_pg_pool_refcount,
300                                         dram_pg_pool_do_release);
301                 } else {
302                         for (i = 0 ; i < phys_pg_pack->npages ; i++) {
303                                 gen_pool_free(vm->dram_pg_pool,
304                                                 phys_pg_pack->pages[i],
305                                                 phys_pg_pack->page_size);
306                                 kref_put(&vm->dram_pg_pool_refcount,
307                                         dram_pg_pool_do_release);
308                         }
309                 }
310         }
311
312         kvfree(phys_pg_pack->pages);
313         kfree(phys_pg_pack);
314 }
315
316 /*
317  * free_device_memory - free device memory
318  *
319  * @ctx                  : current context
320  * @handle              : handle of the memory chunk to free
321  *
322  * This function does the following:
323  * - Free the device memory related to the given handle
324  */
325 static int free_device_memory(struct hl_ctx *ctx, u32 handle)
326 {
327         struct hl_device *hdev = ctx->hdev;
328         struct hl_vm *vm = &hdev->vm;
329         struct hl_vm_phys_pg_pack *phys_pg_pack;
330
331         spin_lock(&vm->idr_lock);
332         phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
333         if (phys_pg_pack) {
334                 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {
335                         dev_err(hdev->dev, "handle %u is mapped, cannot free\n",
336                                 handle);
337                         spin_unlock(&vm->idr_lock);
338                         return -EINVAL;
339                 }
340
341                 /*
342                  * must remove from idr before the freeing of the physical
343                  * pages as the refcount of the pool is also the trigger of the
344                  * idr destroy
345                  */
346                 idr_remove(&vm->phys_pg_pack_handles, handle);
347                 spin_unlock(&vm->idr_lock);
348
349                 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);
350                 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);
351
352                 free_phys_pg_pack(hdev, phys_pg_pack);
353         } else {
354                 spin_unlock(&vm->idr_lock);
355                 dev_err(hdev->dev,
356                         "free device memory failed, no match for handle %u\n",
357                         handle);
358                 return -EINVAL;
359         }
360
361         return 0;
362 }
363
364 /*
365  * clear_va_list_locked - free virtual addresses list
366  *
367  * @hdev                : habanalabs device structure
368  * @va_list             : list of virtual addresses to free
369  *
370  * This function does the following:
371  * - Iterate over the list and free each virtual addresses block
372  *
373  * This function should be called only when va_list lock is taken
374  */
375 static void clear_va_list_locked(struct hl_device *hdev,
376                 struct list_head *va_list)
377 {
378         struct hl_vm_va_block *va_block, *tmp;
379
380         list_for_each_entry_safe(va_block, tmp, va_list, node) {
381                 list_del(&va_block->node);
382                 kfree(va_block);
383         }
384 }
385
386 /*
387  * print_va_list_locked    - print virtual addresses list
388  *
389  * @hdev                : habanalabs device structure
390  * @va_list             : list of virtual addresses to print
391  *
392  * This function does the following:
393  * - Iterate over the list and print each virtual addresses block
394  *
395  * This function should be called only when va_list lock is taken
396  */
397 static void print_va_list_locked(struct hl_device *hdev,
398                 struct list_head *va_list)
399 {
400 #if HL_MMU_DEBUG
401         struct hl_vm_va_block *va_block;
402
403         dev_dbg(hdev->dev, "print va list:\n");
404
405         list_for_each_entry(va_block, va_list, node)
406                 dev_dbg(hdev->dev,
407                         "va block, start: 0x%llx, end: 0x%llx, size: %llu\n",
408                         va_block->start, va_block->end, va_block->size);
409 #endif
410 }
411
412 /*
413  * merge_va_blocks_locked - merge a virtual block if possible
414  *
415  * @hdev                : pointer to the habanalabs device structure
416  * @va_list             : pointer to the virtual addresses block list
417  * @va_block            : virtual block to merge with adjacent blocks
418  *
419  * This function does the following:
420  * - Merge the given blocks with the adjacent blocks if their virtual ranges
421  *   create a contiguous virtual range
422  *
423  * This Function should be called only when va_list lock is taken
424  */
425 static void merge_va_blocks_locked(struct hl_device *hdev,
426                 struct list_head *va_list, struct hl_vm_va_block *va_block)
427 {
428         struct hl_vm_va_block *prev, *next;
429
430         prev = list_prev_entry(va_block, node);
431         if (&prev->node != va_list && prev->end + 1 == va_block->start) {
432                 prev->end = va_block->end;
433                 prev->size = prev->end - prev->start;
434                 list_del(&va_block->node);
435                 kfree(va_block);
436                 va_block = prev;
437         }
438
439         next = list_next_entry(va_block, node);
440         if (&next->node != va_list && va_block->end + 1 == next->start) {
441                 next->start = va_block->start;
442                 next->size = next->end - next->start;
443                 list_del(&va_block->node);
444                 kfree(va_block);
445         }
446 }
447
448 /*
449  * add_va_block_locked - add a virtual block to the virtual addresses list
450  *
451  * @hdev                : pointer to the habanalabs device structure
452  * @va_list             : pointer to the virtual addresses block list
453  * @start               : start virtual address
454  * @end                 : end virtual address
455  *
456  * This function does the following:
457  * - Add the given block to the virtual blocks list and merge with other
458  * blocks if a contiguous virtual block can be created
459  *
460  * This Function should be called only when va_list lock is taken
461  */
462 static int add_va_block_locked(struct hl_device *hdev,
463                 struct list_head *va_list, u64 start, u64 end)
464 {
465         struct hl_vm_va_block *va_block, *res = NULL;
466         u64 size = end - start;
467
468         print_va_list_locked(hdev, va_list);
469
470         list_for_each_entry(va_block, va_list, node) {
471                 /* TODO: remove upon matureness */
472                 if (hl_mem_area_crosses_range(start, size, va_block->start,
473                                 va_block->end)) {
474                         dev_err(hdev->dev,
475                                 "block crossing ranges at start 0x%llx, end 0x%llx\n",
476                                 va_block->start, va_block->end);
477                         return -EINVAL;
478                 }
479
480                 if (va_block->end < start)
481                         res = va_block;
482         }
483
484         va_block = kmalloc(sizeof(*va_block), GFP_KERNEL);
485         if (!va_block)
486                 return -ENOMEM;
487
488         va_block->start = start;
489         va_block->end = end;
490         va_block->size = size;
491
492         if (!res)
493                 list_add(&va_block->node, va_list);
494         else
495                 list_add(&va_block->node, &res->node);
496
497         merge_va_blocks_locked(hdev, va_list, va_block);
498
499         print_va_list_locked(hdev, va_list);
500
501         return 0;
502 }
503
504 /*
505  * add_va_block - wrapper for add_va_block_locked
506  *
507  * @hdev                : pointer to the habanalabs device structure
508  * @va_list             : pointer to the virtual addresses block list
509  * @start               : start virtual address
510  * @end                 : end virtual address
511  *
512  * This function does the following:
513  * - Takes the list lock and calls add_va_block_locked
514  */
515 static inline int add_va_block(struct hl_device *hdev,
516                 struct hl_va_range *va_range, u64 start, u64 end)
517 {
518         int rc;
519
520         mutex_lock(&va_range->lock);
521         rc = add_va_block_locked(hdev, &va_range->list, start, end);
522         mutex_unlock(&va_range->lock);
523
524         return rc;
525 }
526
527 /*
528  * get_va_block() - get a virtual block for the given size and alignment.
529  * @hdev: pointer to the habanalabs device structure.
530  * @va_range: pointer to the virtual addresses range.
531  * @size: requested block size.
532  * @hint_addr: hint for requested address by the user.
533  * @va_block_align: required alignment of the virtual block start address.
534  *
535  * This function does the following:
536  * - Iterate on the virtual block list to find a suitable virtual block for the
537  *   given size and alignment.
538  * - Reserve the requested block and update the list.
539  * - Return the start address of the virtual block.
540  */
541 static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range,
542                         u64 size, u64 hint_addr, u32 va_block_align)
543 {
544         struct hl_vm_va_block *va_block, *new_va_block = NULL;
545         u64 valid_start, valid_size, prev_start, prev_end, align_mask,
546                 res_valid_start = 0, res_valid_size = 0;
547         bool add_prev = false;
548
549         align_mask = ~((u64)va_block_align - 1);
550
551         /* check if hint_addr is aligned */
552         if (hint_addr & (va_block_align - 1))
553                 hint_addr = 0;
554
555         mutex_lock(&va_range->lock);
556
557         print_va_list_locked(hdev, &va_range->list);
558
559         list_for_each_entry(va_block, &va_range->list, node) {
560                 /* calc the first possible aligned addr */
561                 valid_start = va_block->start;
562
563                 if (valid_start & (va_block_align - 1)) {
564                         valid_start &= align_mask;
565                         valid_start += va_block_align;
566                         if (valid_start > va_block->end)
567                                 continue;
568                 }
569
570                 valid_size = va_block->end - valid_start;
571
572                 if (valid_size >= size &&
573                         (!new_va_block || valid_size < res_valid_size)) {
574                         new_va_block = va_block;
575                         res_valid_start = valid_start;
576                         res_valid_size = valid_size;
577                 }
578
579                 if (hint_addr && hint_addr >= valid_start &&
580                                 ((hint_addr + size) <= va_block->end)) {
581                         new_va_block = va_block;
582                         res_valid_start = hint_addr;
583                         res_valid_size = valid_size;
584                         break;
585                 }
586         }
587
588         if (!new_va_block) {
589                 dev_err(hdev->dev, "no available va block for size %llu\n",
590                                 size);
591                 goto out;
592         }
593
594         if (res_valid_start > new_va_block->start) {
595                 prev_start = new_va_block->start;
596                 prev_end = res_valid_start - 1;
597
598                 new_va_block->start = res_valid_start;
599                 new_va_block->size = res_valid_size;
600
601                 add_prev = true;
602         }
603
604         if (new_va_block->size > size) {
605                 new_va_block->start += size;
606                 new_va_block->size = new_va_block->end - new_va_block->start;
607         } else {
608                 list_del(&new_va_block->node);
609                 kfree(new_va_block);
610         }
611
612         if (add_prev)
613                 add_va_block_locked(hdev, &va_range->list, prev_start,
614                                 prev_end);
615
616         print_va_list_locked(hdev, &va_range->list);
617 out:
618         mutex_unlock(&va_range->lock);
619
620         return res_valid_start;
621 }
622
623 /*
624  * hl_reserve_va_block() - reserve a virtual block of a given size.
625  * @hdev: pointer to the habanalabs device structure.
626  * @ctx: current context
627  * @type: virtual addresses range type.
628  * @size: requested block size.
629  * @alignment: required alignment in bytes of the virtual block start address,
630  *             0 means no alignment.
631  *
632  * This function does the following:
633  * - Iterate on the virtual block list to find a suitable virtual block for the
634  *   given size and alignment.
635  * - Reserve the requested block and update the list.
636  * - Return the start address of the virtual block.
637  */
638 u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
639                 enum hl_va_range_type type, u32 size, u32 alignment)
640 {
641         return get_va_block(hdev, ctx->va_range[type], size, 0,
642                         max(alignment, ctx->va_range[type]->page_size));
643 }
644
645 /**
646  * hl_get_va_range_type() - get va_range type for the given address and size.
647  * @address: The start address of the area we want to validate.
648  * @size: The size in bytes of the area we want to validate.
649  * @type: returned va_range type
650  *
651  * Return: true if the area is inside a valid range, false otherwise.
652  */
653 static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size,
654                         enum hl_va_range_type *type)
655 {
656         int i;
657
658         for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX; i++) {
659                 if (hl_mem_area_inside_range(address, size,
660                                 ctx->va_range[i]->start_addr,
661                                 ctx->va_range[i]->end_addr)) {
662                         *type = i;
663                         return 0;
664                 }
665         }
666
667         return -EINVAL;
668 }
669
670 /*
671  * hl_unreserve_va_block - wrapper for add_va_block for unreserving a va block
672  *
673  * @hdev: pointer to the habanalabs device structure
674  * @ctx: current context
675  * @start: start virtual address
676  * @end: end virtual address
677  *
678  * This function does the following:
679  * - Takes the list lock and calls add_va_block_locked
680  */
681 int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
682                 u64 start_addr, u64 size)
683 {
684         enum hl_va_range_type type;
685         int rc;
686
687         rc = hl_get_va_range_type(ctx, start_addr, size, &type);
688         if (rc) {
689                 dev_err(hdev->dev,
690                         "cannot find va_range for va %#llx size %llu",
691                         start_addr, size);
692                 return rc;
693         }
694
695         rc = add_va_block(hdev, ctx->va_range[type], start_addr,
696                                                 start_addr + size - 1);
697         if (rc)
698                 dev_warn(hdev->dev,
699                         "add va block failed for vaddr: 0x%llx\n", start_addr);
700
701         return rc;
702 }
703
704 /*
705  * get_sg_info - get number of pages and the DMA address from SG list
706  *
707  * @sg                 : the SG list
708  * @dma_addr           : pointer to DMA address to return
709  *
710  * Calculate the number of consecutive pages described by the SG list. Take the
711  * offset of the address in the first page, add to it the length and round it up
712  * to the number of needed pages.
713  */
714 static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
715 {
716         *dma_addr = sg_dma_address(sg);
717
718         return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
719                         (PAGE_SIZE - 1)) >> PAGE_SHIFT;
720 }
721
722 /*
723  * init_phys_pg_pack_from_userptr - initialize physical page pack from host
724  *                                  memory
725  * @ctx: current context
726  * @userptr: userptr to initialize from
727  * @pphys_pg_pack: result pointer
728  *
729  * This function does the following:
730  * - Pin the physical pages related to the given virtual block
731  * - Create a physical page pack from the physical pages related to the given
732  *   virtual block
733  */
734 static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx,
735                                 struct hl_userptr *userptr,
736                                 struct hl_vm_phys_pg_pack **pphys_pg_pack)
737 {
738         struct hl_vm_phys_pg_pack *phys_pg_pack;
739         struct scatterlist *sg;
740         dma_addr_t dma_addr;
741         u64 page_mask, total_npages;
742         u32 npages, page_size = PAGE_SIZE,
743                 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size;
744         bool first = true, is_huge_page_opt = true;
745         int rc, i, j;
746         u32 pgs_in_huge_page = huge_page_size >> __ffs(page_size);
747
748         phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);
749         if (!phys_pg_pack)
750                 return -ENOMEM;
751
752         phys_pg_pack->vm_type = userptr->vm_type;
753         phys_pg_pack->created_from_userptr = true;
754         phys_pg_pack->asid = ctx->asid;
755         atomic_set(&phys_pg_pack->mapping_cnt, 1);
756
757         /* Only if all dma_addrs are aligned to 2MB and their
758          * sizes is at least 2MB, we can use huge page mapping.
759          * We limit the 2MB optimization to this condition,
760          * since later on we acquire the related VA range as one
761          * consecutive block.
762          */
763         total_npages = 0;
764         for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
765                 npages = get_sg_info(sg, &dma_addr);
766
767                 total_npages += npages;
768
769                 if ((npages % pgs_in_huge_page) ||
770                                         (dma_addr & (huge_page_size - 1)))
771                         is_huge_page_opt = false;
772         }
773
774         if (is_huge_page_opt) {
775                 page_size = huge_page_size;
776                 do_div(total_npages, pgs_in_huge_page);
777         }
778
779         page_mask = ~(((u64) page_size) - 1);
780
781         phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),
782                                                 GFP_KERNEL);
783         if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {
784                 rc = -ENOMEM;
785                 goto page_pack_arr_mem_err;
786         }
787
788         phys_pg_pack->npages = total_npages;
789         phys_pg_pack->page_size = page_size;
790         phys_pg_pack->total_size = total_npages * page_size;
791
792         j = 0;
793         for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) {
794                 npages = get_sg_info(sg, &dma_addr);
795
796                 /* align down to physical page size and save the offset */
797                 if (first) {
798                         first = false;
799                         phys_pg_pack->offset = dma_addr & (page_size - 1);
800                         dma_addr &= page_mask;
801                 }
802
803                 while (npages) {
804                         phys_pg_pack->pages[j++] = dma_addr;
805                         dma_addr += page_size;
806
807                         if (is_huge_page_opt)
808                                 npages -= pgs_in_huge_page;
809                         else
810                                 npages--;
811                 }
812         }
813
814         *pphys_pg_pack = phys_pg_pack;
815
816         return 0;
817
818 page_pack_arr_mem_err:
819         kfree(phys_pg_pack);
820
821         return rc;
822 }
823
824 /*
825  * map_phys_pg_pack - maps the physical page pack.
826  * @ctx: current context
827  * @vaddr: start address of the virtual area to map from
828  * @phys_pg_pack: the pack of physical pages to map to
829  *
830  * This function does the following:
831  * - Maps each chunk of virtual memory to matching physical chunk
832  * - Stores number of successful mappings in the given argument
833  * - Returns 0 on success, error code otherwise
834  */
835 static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
836                                 struct hl_vm_phys_pg_pack *phys_pg_pack)
837 {
838         struct hl_device *hdev = ctx->hdev;
839         u64 next_vaddr = vaddr, paddr, mapped_pg_cnt = 0, i;
840         u32 page_size = phys_pg_pack->page_size;
841         int rc = 0;
842
843         for (i = 0 ; i < phys_pg_pack->npages ; i++) {
844                 paddr = phys_pg_pack->pages[i];
845
846                 rc = hl_mmu_map_page(ctx, next_vaddr, paddr, page_size,
847                                 (i + 1) == phys_pg_pack->npages);
848                 if (rc) {
849                         dev_err(hdev->dev,
850                                 "map failed for handle %u, npages: %llu, mapped: %llu",
851                                 phys_pg_pack->handle, phys_pg_pack->npages,
852                                 mapped_pg_cnt);
853                         goto err;
854                 }
855
856                 mapped_pg_cnt++;
857                 next_vaddr += page_size;
858         }
859
860         return 0;
861
862 err:
863         next_vaddr = vaddr;
864         for (i = 0 ; i < mapped_pg_cnt ; i++) {
865                 if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
866                                         (i + 1) == mapped_pg_cnt))
867                         dev_warn_ratelimited(hdev->dev,
868                                 "failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n",
869                                         phys_pg_pack->handle, next_vaddr,
870                                         phys_pg_pack->pages[i], page_size);
871
872                 next_vaddr += page_size;
873         }
874
875         return rc;
876 }
877
878 /*
879  * unmap_phys_pg_pack - unmaps the physical page pack
880  * @ctx: current context
881  * @vaddr: start address of the virtual area to unmap
882  * @phys_pg_pack: the pack of physical pages to unmap
883  */
884 static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
885                                 struct hl_vm_phys_pg_pack *phys_pg_pack)
886 {
887         struct hl_device *hdev = ctx->hdev;
888         u64 next_vaddr, i;
889         bool is_host_addr;
890         u32 page_size;
891
892         is_host_addr = !hl_is_dram_va(hdev, vaddr);
893         page_size = phys_pg_pack->page_size;
894         next_vaddr = vaddr;
895
896         for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {
897                 if (hl_mmu_unmap_page(ctx, next_vaddr, page_size,
898                                        (i + 1) == phys_pg_pack->npages))
899                         dev_warn_ratelimited(hdev->dev,
900                         "unmap failed for vaddr: 0x%llx\n", next_vaddr);
901
902                 /*
903                  * unmapping on Palladium can be really long, so avoid a CPU
904                  * soft lockup bug by sleeping a little between unmapping pages
905                  *
906                  * In addition, when unmapping host memory we pass through
907                  * the Linux kernel to unpin the pages and that takes a long
908                  * time. Therefore, sleep every 32K pages to avoid soft lockup
909                  */
910                 if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0))
911                         usleep_range(50, 200);
912         }
913 }
914
915 static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args,
916                                 u64 *paddr)
917 {
918         struct hl_device *hdev = ctx->hdev;
919         struct hl_vm *vm = &hdev->vm;
920         struct hl_vm_phys_pg_pack *phys_pg_pack;
921         u32 handle;
922
923         handle = lower_32_bits(args->map_device.handle);
924         spin_lock(&vm->idr_lock);
925         phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
926         if (!phys_pg_pack) {
927                 spin_unlock(&vm->idr_lock);
928                 dev_err(hdev->dev, "no match for handle %u\n", handle);
929                 return -EINVAL;
930         }
931
932         *paddr = phys_pg_pack->pages[0];
933
934         spin_unlock(&vm->idr_lock);
935
936         return 0;
937 }
938
939 /*
940  * map_device_va - map the given memory
941  *
942  * @ctx          : current context
943  * @args         : host parameters with handle/host virtual address
944  * @device_addr  : pointer to result device virtual address
945  *
946  * This function does the following:
947  * - If given a physical device memory handle, map to a device virtual block
948  *   and return the start address of this block
949  * - If given a host virtual address and size, find the related physical pages,
950  *   map a device virtual block to this pages and return the start address of
951  *   this block
952  */
953 static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
954                 u64 *device_addr)
955 {
956         struct hl_device *hdev = ctx->hdev;
957         struct hl_vm *vm = &hdev->vm;
958         struct hl_vm_phys_pg_pack *phys_pg_pack;
959         struct hl_userptr *userptr = NULL;
960         struct hl_vm_hash_node *hnode;
961         struct hl_va_range *va_range;
962         enum vm_type_t *vm_type;
963         u64 ret_vaddr, hint_addr;
964         u32 handle = 0, va_block_align;
965         int rc;
966         bool is_userptr = args->flags & HL_MEM_USERPTR;
967
968         /* Assume failure */
969         *device_addr = 0;
970
971         if (is_userptr) {
972                 u64 addr = args->map_host.host_virt_addr,
973                         size = args->map_host.mem_size;
974                 u32 page_size = hdev->asic_prop.pmmu.page_size,
975                         huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
976
977                 rc = dma_map_host_va(hdev, addr, size, &userptr);
978                 if (rc) {
979                         dev_err(hdev->dev, "failed to get userptr from va\n");
980                         return rc;
981                 }
982
983                 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
984                                 &phys_pg_pack);
985                 if (rc) {
986                         dev_err(hdev->dev,
987                                 "unable to init page pack for vaddr 0x%llx\n",
988                                 addr);
989                         goto init_page_pack_err;
990                 }
991
992                 vm_type = (enum vm_type_t *) userptr;
993                 hint_addr = args->map_host.hint_addr;
994                 handle = phys_pg_pack->handle;
995
996                 /* get required alignment */
997                 if (phys_pg_pack->page_size == page_size) {
998                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
999
1000                         /*
1001                          * huge page alignment may be needed in case of regular
1002                          * page mapping, depending on the host VA alignment
1003                          */
1004                         if (addr & (huge_page_size - 1))
1005                                 va_block_align = page_size;
1006                         else
1007                                 va_block_align = huge_page_size;
1008                 } else {
1009                         /*
1010                          * huge page alignment is needed in case of huge page
1011                          * mapping
1012                          */
1013                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1014                         va_block_align = huge_page_size;
1015                 }
1016         } else {
1017                 handle = lower_32_bits(args->map_device.handle);
1018
1019                 spin_lock(&vm->idr_lock);
1020                 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
1021                 if (!phys_pg_pack) {
1022                         spin_unlock(&vm->idr_lock);
1023                         dev_err(hdev->dev,
1024                                 "no match for handle %u\n", handle);
1025                         return -EINVAL;
1026                 }
1027
1028                 /* increment now to avoid freeing device memory while mapping */
1029                 atomic_inc(&phys_pg_pack->mapping_cnt);
1030
1031                 spin_unlock(&vm->idr_lock);
1032
1033                 vm_type = (enum vm_type_t *) phys_pg_pack;
1034
1035                 hint_addr = args->map_device.hint_addr;
1036
1037                 /* DRAM VA alignment is the same as the DRAM page size */
1038                 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1039                 va_block_align = hdev->asic_prop.dmmu.page_size;
1040         }
1041
1042         /*
1043          * relevant for mapping device physical memory only, as host memory is
1044          * implicitly shared
1045          */
1046         if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&
1047                         phys_pg_pack->asid != ctx->asid) {
1048                 dev_err(hdev->dev,
1049                         "Failed to map memory, handle %u is not shared\n",
1050                         handle);
1051                 rc = -EPERM;
1052                 goto shared_err;
1053         }
1054
1055         hnode = kzalloc(sizeof(*hnode), GFP_KERNEL);
1056         if (!hnode) {
1057                 rc = -ENOMEM;
1058                 goto hnode_err;
1059         }
1060
1061         ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
1062                                         hint_addr, va_block_align);
1063         if (!ret_vaddr) {
1064                 dev_err(hdev->dev, "no available va block for handle %u\n",
1065                                 handle);
1066                 rc = -ENOMEM;
1067                 goto va_block_err;
1068         }
1069
1070         mutex_lock(&ctx->mmu_lock);
1071
1072         rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
1073         if (rc) {
1074                 mutex_unlock(&ctx->mmu_lock);
1075                 dev_err(hdev->dev, "mapping page pack failed for handle %u\n",
1076                                 handle);
1077                 goto map_err;
1078         }
1079
1080         rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
1081
1082         mutex_unlock(&ctx->mmu_lock);
1083
1084         if (rc) {
1085                 dev_err(hdev->dev,
1086                         "mapping handle %u failed due to MMU cache invalidation\n",
1087                         handle);
1088                 goto map_err;
1089         }
1090
1091         ret_vaddr += phys_pg_pack->offset;
1092
1093         hnode->ptr = vm_type;
1094         hnode->vaddr = ret_vaddr;
1095
1096         mutex_lock(&ctx->mem_hash_lock);
1097         hash_add(ctx->mem_hash, &hnode->node, ret_vaddr);
1098         mutex_unlock(&ctx->mem_hash_lock);
1099
1100         *device_addr = ret_vaddr;
1101
1102         if (is_userptr)
1103                 free_phys_pg_pack(hdev, phys_pg_pack);
1104
1105         return 0;
1106
1107 map_err:
1108         if (add_va_block(hdev, va_range, ret_vaddr,
1109                                 ret_vaddr + phys_pg_pack->total_size - 1))
1110                 dev_warn(hdev->dev,
1111                         "release va block failed for handle 0x%x, vaddr: 0x%llx\n",
1112                                 handle, ret_vaddr);
1113
1114 va_block_err:
1115         kfree(hnode);
1116 hnode_err:
1117 shared_err:
1118         atomic_dec(&phys_pg_pack->mapping_cnt);
1119         if (is_userptr)
1120                 free_phys_pg_pack(hdev, phys_pg_pack);
1121 init_page_pack_err:
1122         if (is_userptr)
1123                 dma_unmap_host_va(hdev, userptr);
1124
1125         return rc;
1126 }
1127
1128 /*
1129  * unmap_device_va      - unmap the given device virtual address
1130  *
1131  * @ctx                 : current context
1132  * @vaddr               : device virtual address to unmap
1133  * @ctx_free            : true if in context free flow, false otherwise.
1134  *
1135  * This function does the following:
1136  * - Unmap the physical pages related to the given virtual address
1137  * - return the device virtual block to the virtual block list
1138  */
1139 static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
1140 {
1141         struct hl_device *hdev = ctx->hdev;
1142         struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
1143         struct hl_vm_hash_node *hnode = NULL;
1144         struct hl_userptr *userptr = NULL;
1145         struct hl_va_range *va_range;
1146         enum vm_type_t *vm_type;
1147         bool is_userptr;
1148         int rc = 0;
1149
1150         /* protect from double entrance */
1151         mutex_lock(&ctx->mem_hash_lock);
1152         hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr)
1153                 if (vaddr == hnode->vaddr)
1154                         break;
1155
1156         if (!hnode) {
1157                 mutex_unlock(&ctx->mem_hash_lock);
1158                 dev_err(hdev->dev,
1159                         "unmap failed, no mem hnode for vaddr 0x%llx\n",
1160                         vaddr);
1161                 return -EINVAL;
1162         }
1163
1164         hash_del(&hnode->node);
1165         mutex_unlock(&ctx->mem_hash_lock);
1166
1167         vm_type = hnode->ptr;
1168
1169         if (*vm_type == VM_TYPE_USERPTR) {
1170                 is_userptr = true;
1171                 userptr = hnode->ptr;
1172                 rc = init_phys_pg_pack_from_userptr(ctx, userptr,
1173                                                         &phys_pg_pack);
1174                 if (rc) {
1175                         dev_err(hdev->dev,
1176                                 "unable to init page pack for vaddr 0x%llx\n",
1177                                 vaddr);
1178                         goto vm_type_err;
1179                 }
1180
1181                 if (phys_pg_pack->page_size ==
1182                                         hdev->asic_prop.pmmu.page_size)
1183                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1184                 else
1185                         va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
1186         } else if (*vm_type == VM_TYPE_PHYS_PACK) {
1187                 is_userptr = false;
1188                 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
1189                 phys_pg_pack = hnode->ptr;
1190         } else {
1191                 dev_warn(hdev->dev,
1192                         "unmap failed, unknown vm desc for vaddr 0x%llx\n",
1193                                 vaddr);
1194                 rc = -EFAULT;
1195                 goto vm_type_err;
1196         }
1197
1198         if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {
1199                 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr);
1200                 rc = -EINVAL;
1201                 goto mapping_cnt_err;
1202         }
1203
1204         vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);
1205
1206         mutex_lock(&ctx->mmu_lock);
1207
1208         unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);
1209
1210         /*
1211          * During context free this function is called in a loop to clean all
1212          * the context mappings. Hence the cache invalidation can be called once
1213          * at the loop end rather than for each iteration
1214          */
1215         if (!ctx_free)
1216                 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
1217                                                                 *vm_type);
1218
1219         mutex_unlock(&ctx->mmu_lock);
1220
1221         /*
1222          * If the context is closing we don't need to check for the MMU cache
1223          * invalidation return code and update the VA free list as in this flow
1224          * we invalidate the MMU cache outside of this unmap function and the VA
1225          * free list will be freed anyway.
1226          */
1227         if (!ctx_free) {
1228                 int tmp_rc;
1229
1230                 if (rc)
1231                         dev_err(hdev->dev,
1232                                 "unmapping vaddr 0x%llx failed due to MMU cache invalidation\n",
1233                                 vaddr);
1234
1235                 tmp_rc = add_va_block(hdev, va_range, vaddr,
1236                                         vaddr + phys_pg_pack->total_size - 1);
1237                 if (tmp_rc) {
1238                         dev_warn(hdev->dev,
1239                                         "add va block failed for vaddr: 0x%llx\n",
1240                                         vaddr);
1241                         if (!rc)
1242                                 rc = tmp_rc;
1243                 }
1244         }
1245
1246         atomic_dec(&phys_pg_pack->mapping_cnt);
1247         kfree(hnode);
1248
1249         if (is_userptr) {
1250                 free_phys_pg_pack(hdev, phys_pg_pack);
1251                 dma_unmap_host_va(hdev, userptr);
1252         }
1253
1254         return rc;
1255
1256 mapping_cnt_err:
1257         if (is_userptr)
1258                 free_phys_pg_pack(hdev, phys_pg_pack);
1259 vm_type_err:
1260         mutex_lock(&ctx->mem_hash_lock);
1261         hash_add(ctx->mem_hash, &hnode->node, vaddr);
1262         mutex_unlock(&ctx->mem_hash_lock);
1263
1264         return rc;
1265 }
1266
1267 static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args)
1268 {
1269         struct hl_device *hdev = hpriv->hdev;
1270         struct hl_ctx *ctx = hpriv->ctx;
1271         u64 device_addr = 0;
1272         u32 handle = 0;
1273         int rc;
1274
1275         switch (args->in.op) {
1276         case HL_MEM_OP_ALLOC:
1277                 if (args->in.alloc.mem_size == 0) {
1278                         dev_err(hdev->dev,
1279                                 "alloc size must be larger than 0\n");
1280                         rc = -EINVAL;
1281                         goto out;
1282                 }
1283
1284                 /* Force contiguous as there are no real MMU
1285                  * translations to overcome physical memory gaps
1286                  */
1287                 args->in.flags |= HL_MEM_CONTIGUOUS;
1288                 rc = alloc_device_memory(ctx, &args->in, &handle);
1289
1290                 memset(args, 0, sizeof(*args));
1291                 args->out.handle = (__u64) handle;
1292                 break;
1293
1294         case HL_MEM_OP_FREE:
1295                 rc = free_device_memory(ctx, args->in.free.handle);
1296                 break;
1297
1298         case HL_MEM_OP_MAP:
1299                 if (args->in.flags & HL_MEM_USERPTR) {
1300                         device_addr = args->in.map_host.host_virt_addr;
1301                         rc = 0;
1302                 } else {
1303                         rc = get_paddr_from_handle(ctx, &args->in,
1304                                         &device_addr);
1305                 }
1306
1307                 memset(args, 0, sizeof(*args));
1308                 args->out.device_virt_addr = device_addr;
1309                 break;
1310
1311         case HL_MEM_OP_UNMAP:
1312                 rc = 0;
1313                 break;
1314
1315         default:
1316                 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1317                 rc = -ENOTTY;
1318                 break;
1319         }
1320
1321 out:
1322         return rc;
1323 }
1324
1325 int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
1326 {
1327         enum hl_device_status status;
1328         union hl_mem_args *args = data;
1329         struct hl_device *hdev = hpriv->hdev;
1330         struct hl_ctx *ctx = hpriv->ctx;
1331         u64 device_addr = 0;
1332         u32 handle = 0;
1333         int rc;
1334
1335         if (!hl_device_operational(hdev, &status)) {
1336                 dev_warn_ratelimited(hdev->dev,
1337                         "Device is %s. Can't execute MEMORY IOCTL\n",
1338                         hdev->status[status]);
1339                 return -EBUSY;
1340         }
1341
1342         if (!hdev->mmu_enable)
1343                 return mem_ioctl_no_mmu(hpriv, args);
1344
1345         switch (args->in.op) {
1346         case HL_MEM_OP_ALLOC:
1347                 if (args->in.alloc.mem_size == 0) {
1348                         dev_err(hdev->dev,
1349                                 "alloc size must be larger than 0\n");
1350                         rc = -EINVAL;
1351                         goto out;
1352                 }
1353
1354                 /* If DRAM does not support virtual memory the driver won't
1355                  * handle the allocation/freeing of that memory. However, for
1356                  * system administration/monitoring purposes, the driver will
1357                  * keep track of the amount of DRAM memory that is allocated
1358                  * and freed by the user. Because this code totally relies on
1359                  * the user's input, the driver can't ensure the validity
1360                  * of this accounting.
1361                  */
1362                 if (!hdev->asic_prop.dram_supports_virtual_memory) {
1363                         atomic64_add(args->in.alloc.mem_size,
1364                                         &ctx->dram_phys_mem);
1365                         atomic64_add(args->in.alloc.mem_size,
1366                                         &hdev->dram_used_mem);
1367
1368                         dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
1369                         rc = 0;
1370
1371                         memset(args, 0, sizeof(*args));
1372                         args->out.handle = 0;
1373                         goto out;
1374                 }
1375
1376                 rc = alloc_device_memory(ctx, &args->in, &handle);
1377
1378                 memset(args, 0, sizeof(*args));
1379                 args->out.handle = (__u64) handle;
1380                 break;
1381
1382         case HL_MEM_OP_FREE:
1383                 /* If DRAM does not support virtual memory the driver won't
1384                  * handle the allocation/freeing of that memory. However, for
1385                  * system administration/monitoring purposes, the driver will
1386                  * keep track of the amount of DRAM memory that is allocated
1387                  * and freed by the user. Because this code totally relies on
1388                  * the user's input, the driver can't ensure the validity
1389                  * of this accounting.
1390                  */
1391                 if (!hdev->asic_prop.dram_supports_virtual_memory) {
1392                         atomic64_sub(args->in.alloc.mem_size,
1393                                         &ctx->dram_phys_mem);
1394                         atomic64_sub(args->in.alloc.mem_size,
1395                                         &hdev->dram_used_mem);
1396
1397                         dev_dbg(hdev->dev, "DRAM alloc is not supported\n");
1398                         rc = 0;
1399
1400                         goto out;
1401                 }
1402
1403                 rc = free_device_memory(ctx, args->in.free.handle);
1404                 break;
1405
1406         case HL_MEM_OP_MAP:
1407                 rc = map_device_va(ctx, &args->in, &device_addr);
1408
1409                 memset(args, 0, sizeof(*args));
1410                 args->out.device_virt_addr = device_addr;
1411                 break;
1412
1413         case HL_MEM_OP_UNMAP:
1414                 rc = unmap_device_va(ctx, args->in.unmap.device_virt_addr,
1415                                         false);
1416                 break;
1417
1418         default:
1419                 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
1420                 rc = -ENOTTY;
1421                 break;
1422         }
1423
1424 out:
1425         return rc;
1426 }
1427
1428 static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
1429                                 u32 npages, u64 start, u32 offset,
1430                                 struct hl_userptr *userptr)
1431 {
1432         int rc;
1433
1434         if (!access_ok((void __user *) (uintptr_t) addr, size)) {
1435                 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr);
1436                 return -EFAULT;
1437         }
1438
1439         userptr->vec = frame_vector_create(npages);
1440         if (!userptr->vec) {
1441                 dev_err(hdev->dev, "Failed to create frame vector\n");
1442                 return -ENOMEM;
1443         }
1444
1445         rc = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
1446                                 userptr->vec);
1447
1448         if (rc != npages) {
1449                 dev_err(hdev->dev,
1450                         "Failed to map host memory, user ptr probably wrong\n");
1451                 if (rc < 0)
1452                         goto destroy_framevec;
1453                 rc = -EFAULT;
1454                 goto put_framevec;
1455         }
1456
1457         if (frame_vector_to_pages(userptr->vec) < 0) {
1458                 dev_err(hdev->dev,
1459                         "Failed to translate frame vector to pages\n");
1460                 rc = -EFAULT;
1461                 goto put_framevec;
1462         }
1463
1464         rc = sg_alloc_table_from_pages(userptr->sgt,
1465                                         frame_vector_pages(userptr->vec),
1466                                         npages, offset, size, GFP_ATOMIC);
1467         if (rc < 0) {
1468                 dev_err(hdev->dev, "failed to create SG table from pages\n");
1469                 goto put_framevec;
1470         }
1471
1472         return 0;
1473
1474 put_framevec:
1475         put_vaddr_frames(userptr->vec);
1476 destroy_framevec:
1477         frame_vector_destroy(userptr->vec);
1478         return rc;
1479 }
1480
1481 /*
1482  * hl_pin_host_memory - pins a chunk of host memory.
1483  * @hdev: pointer to the habanalabs device structure
1484  * @addr: the host virtual address of the memory area
1485  * @size: the size of the memory area
1486  * @userptr: pointer to hl_userptr structure
1487  *
1488  * This function does the following:
1489  * - Pins the physical pages
1490  * - Create an SG list from those pages
1491  */
1492 int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
1493                                         struct hl_userptr *userptr)
1494 {
1495         u64 start, end;
1496         u32 npages, offset;
1497         int rc;
1498
1499         if (!size) {
1500                 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size);
1501                 return -EINVAL;
1502         }
1503
1504         /*
1505          * If the combination of the address and size requested for this memory
1506          * region causes an integer overflow, return error.
1507          */
1508         if (((addr + size) < addr) ||
1509                         PAGE_ALIGN(addr + size) < (addr + size)) {
1510                 dev_err(hdev->dev,
1511                         "user pointer 0x%llx + %llu causes integer overflow\n",
1512                         addr, size);
1513                 return -EINVAL;
1514         }
1515
1516         /*
1517          * This function can be called also from data path, hence use atomic
1518          * always as it is not a big allocation.
1519          */
1520         userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_ATOMIC);
1521         if (!userptr->sgt)
1522                 return -ENOMEM;
1523
1524         start = addr & PAGE_MASK;
1525         offset = addr & ~PAGE_MASK;
1526         end = PAGE_ALIGN(addr + size);
1527         npages = (end - start) >> PAGE_SHIFT;
1528
1529         userptr->size = size;
1530         userptr->addr = addr;
1531         userptr->dma_mapped = false;
1532         INIT_LIST_HEAD(&userptr->job_node);
1533
1534         rc = get_user_memory(hdev, addr, size, npages, start, offset,
1535                                 userptr);
1536         if (rc) {
1537                 dev_err(hdev->dev,
1538                         "failed to get user memory for address 0x%llx\n",
1539                         addr);
1540                 goto free_sgt;
1541         }
1542
1543         hl_debugfs_add_userptr(hdev, userptr);
1544
1545         return 0;
1546
1547 free_sgt:
1548         kfree(userptr->sgt);
1549         return rc;
1550 }
1551
1552 /*
1553  * hl_unpin_host_memory - unpins a chunk of host memory.
1554  * @hdev: pointer to the habanalabs device structure
1555  * @userptr: pointer to hl_userptr structure
1556  *
1557  * This function does the following:
1558  * - Unpins the physical pages related to the host memory
1559  * - Free the SG list
1560  */
1561 void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
1562 {
1563         struct page **pages;
1564
1565         hl_debugfs_remove_userptr(hdev, userptr);
1566
1567         if (userptr->dma_mapped)
1568                 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl,
1569                                                         userptr->sgt->nents,
1570                                                         userptr->dir);
1571
1572         pages = frame_vector_pages(userptr->vec);
1573         if (!IS_ERR(pages)) {
1574                 int i;
1575
1576                 for (i = 0; i < frame_vector_count(userptr->vec); i++)
1577                         set_page_dirty_lock(pages[i]);
1578         }
1579         put_vaddr_frames(userptr->vec);
1580         frame_vector_destroy(userptr->vec);
1581
1582         list_del(&userptr->job_node);
1583
1584         sg_free_table(userptr->sgt);
1585         kfree(userptr->sgt);
1586 }
1587
1588 /*
1589  * hl_userptr_delete_list - clear userptr list
1590  *
1591  * @hdev                : pointer to the habanalabs device structure
1592  * @userptr_list        : pointer to the list to clear
1593  *
1594  * This function does the following:
1595  * - Iterates over the list and unpins the host memory and frees the userptr
1596  *   structure.
1597  */
1598 void hl_userptr_delete_list(struct hl_device *hdev,
1599                                 struct list_head *userptr_list)
1600 {
1601         struct hl_userptr *userptr, *tmp;
1602
1603         list_for_each_entry_safe(userptr, tmp, userptr_list, job_node) {
1604                 hl_unpin_host_memory(hdev, userptr);
1605                 kfree(userptr);
1606         }
1607
1608         INIT_LIST_HEAD(userptr_list);
1609 }
1610
1611 /*
1612  * hl_userptr_is_pinned - returns whether the given userptr is pinned
1613  *
1614  * @hdev                : pointer to the habanalabs device structure
1615  * @userptr_list        : pointer to the list to clear
1616  * @userptr             : pointer to userptr to check
1617  *
1618  * This function does the following:
1619  * - Iterates over the list and checks if the given userptr is in it, means is
1620  *   pinned. If so, returns true, otherwise returns false.
1621  */
1622 bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr,
1623                                 u32 size, struct list_head *userptr_list,
1624                                 struct hl_userptr **userptr)
1625 {
1626         list_for_each_entry((*userptr), userptr_list, job_node) {
1627                 if ((addr == (*userptr)->addr) && (size == (*userptr)->size))
1628                         return true;
1629         }
1630
1631         return false;
1632 }
1633
1634 /*
1635  * va_range_init - initialize virtual addresses range
1636  * @hdev: pointer to the habanalabs device structure
1637  * @va_range: pointer to the range to initialize
1638  * @start: range start address
1639  * @end: range end address
1640  *
1641  * This function does the following:
1642  * - Initializes the virtual addresses list of the given range with the given
1643  *   addresses.
1644  */
1645 static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range,
1646                                 u64 start, u64 end, u32 page_size)
1647 {
1648         int rc;
1649
1650         INIT_LIST_HEAD(&va_range->list);
1651
1652         /* PAGE_SIZE alignment */
1653
1654         if (start & (PAGE_SIZE - 1)) {
1655                 start &= PAGE_MASK;
1656                 start += PAGE_SIZE;
1657         }
1658
1659         if (end & (PAGE_SIZE - 1))
1660                 end &= PAGE_MASK;
1661
1662         if (start >= end) {
1663                 dev_err(hdev->dev, "too small vm range for va list\n");
1664                 return -EFAULT;
1665         }
1666
1667         rc = add_va_block(hdev, va_range, start, end);
1668
1669         if (rc) {
1670                 dev_err(hdev->dev, "Failed to init host va list\n");
1671                 return rc;
1672         }
1673
1674         va_range->start_addr = start;
1675         va_range->end_addr = end;
1676         va_range->page_size = page_size;
1677
1678         return 0;
1679 }
1680
1681 /*
1682  * va_range_fini() - clear a virtual addresses range
1683  * @hdev: pointer to the habanalabs structure
1684  * va_range: pointer to virtual addresses range
1685  *
1686  * This function does the following:
1687  * - Frees the virtual addresses block list and its lock
1688  */
1689 static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range)
1690 {
1691         mutex_lock(&va_range->lock);
1692         clear_va_list_locked(hdev, &va_range->list);
1693         mutex_unlock(&va_range->lock);
1694
1695         mutex_destroy(&va_range->lock);
1696         kfree(va_range);
1697 }
1698
1699 /*
1700  * vm_ctx_init_with_ranges() - initialize virtual memory for context
1701  * @ctx: pointer to the habanalabs context structure
1702  * @host_range_start: host virtual addresses range start.
1703  * @host_range_end: host virtual addresses range end.
1704  * @host_huge_range_start: host virtual addresses range start for memory
1705  *                          allocated with huge pages.
1706  * @host_huge_range_end: host virtual addresses range end for memory allocated
1707  *                        with huge pages.
1708  * @dram_range_start: dram virtual addresses range start.
1709  * @dram_range_end: dram virtual addresses range end.
1710  *
1711  * This function initializes the following:
1712  * - MMU for context
1713  * - Virtual address to area descriptor hashtable
1714  * - Virtual block list of available virtual memory
1715  */
1716 static int vm_ctx_init_with_ranges(struct hl_ctx *ctx,
1717                                         u64 host_range_start,
1718                                         u64 host_range_end,
1719                                         u32 host_page_size,
1720                                         u64 host_huge_range_start,
1721                                         u64 host_huge_range_end,
1722                                         u32 host_huge_page_size,
1723                                         u64 dram_range_start,
1724                                         u64 dram_range_end,
1725                                         u32 dram_page_size)
1726 {
1727         struct hl_device *hdev = ctx->hdev;
1728         int i, rc;
1729
1730         for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++) {
1731                 ctx->va_range[i] =
1732                         kzalloc(sizeof(struct hl_va_range), GFP_KERNEL);
1733                 if (!ctx->va_range[i]) {
1734                         rc = -ENOMEM;
1735                         goto free_va_range;
1736                 }
1737         }
1738
1739         rc = hl_mmu_ctx_init(ctx);
1740         if (rc) {
1741                 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid);
1742                 goto free_va_range;
1743         }
1744
1745         mutex_init(&ctx->mem_hash_lock);
1746         hash_init(ctx->mem_hash);
1747
1748         mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1749
1750         rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST],
1751                         host_range_start, host_range_end, host_page_size);
1752         if (rc) {
1753                 dev_err(hdev->dev, "failed to init host vm range\n");
1754                 goto mmu_ctx_fini;
1755         }
1756
1757         if (hdev->pmmu_huge_range) {
1758                 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1759
1760                 rc = va_range_init(hdev,
1761                         ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE],
1762                         host_huge_range_start, host_huge_range_end,
1763                         host_huge_page_size);
1764                 if (rc) {
1765                         dev_err(hdev->dev,
1766                                 "failed to init host huge vm range\n");
1767                         goto clear_host_va_range;
1768                 }
1769         } else {
1770                 kfree(ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
1771                 ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] =
1772                                 ctx->va_range[HL_VA_RANGE_TYPE_HOST];
1773         }
1774
1775         mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
1776
1777         rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM],
1778                         dram_range_start, dram_range_end, dram_page_size);
1779         if (rc) {
1780                 dev_err(hdev->dev, "failed to init dram vm range\n");
1781                 goto clear_host_huge_va_range;
1782         }
1783
1784         hl_debugfs_add_ctx_mem_hash(hdev, ctx);
1785
1786         return 0;
1787
1788 clear_host_huge_va_range:
1789         mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock);
1790
1791         if (hdev->pmmu_huge_range) {
1792                 mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1793                 clear_va_list_locked(hdev,
1794                         &ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list);
1795                 mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1796         }
1797 clear_host_va_range:
1798         if (hdev->pmmu_huge_range)
1799                 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock);
1800         mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1801         clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list);
1802         mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1803 mmu_ctx_fini:
1804         mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock);
1805         mutex_destroy(&ctx->mem_hash_lock);
1806         hl_mmu_ctx_fini(ctx);
1807 free_va_range:
1808         for (i = 0 ; i < HL_VA_RANGE_TYPE_MAX ; i++)
1809                 kfree(ctx->va_range[i]);
1810
1811         return rc;
1812 }
1813
1814 int hl_vm_ctx_init(struct hl_ctx *ctx)
1815 {
1816         struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
1817         u64 host_range_start, host_range_end, host_huge_range_start,
1818                 host_huge_range_end, dram_range_start, dram_range_end;
1819         u32 host_page_size, host_huge_page_size, dram_page_size;
1820
1821         atomic64_set(&ctx->dram_phys_mem, 0);
1822
1823         /*
1824          * - If MMU is enabled, init the ranges as usual.
1825          * - If MMU is disabled, in case of host mapping, the returned address
1826          *   is the given one.
1827          *   In case of DRAM mapping, the returned address is the physical
1828          *   address of the memory related to the given handle.
1829          */
1830         if (!ctx->hdev->mmu_enable)
1831                 return 0;
1832
1833         dram_range_start = prop->dmmu.start_addr;
1834         dram_range_end = prop->dmmu.end_addr;
1835         dram_page_size = prop->dmmu.page_size;
1836         host_range_start = prop->pmmu.start_addr;
1837         host_range_end = prop->pmmu.end_addr;
1838         host_page_size = prop->pmmu.page_size;
1839         host_huge_range_start = prop->pmmu_huge.start_addr;
1840         host_huge_range_end = prop->pmmu_huge.end_addr;
1841         host_huge_page_size = prop->pmmu_huge.page_size;
1842
1843         return vm_ctx_init_with_ranges(ctx, host_range_start, host_range_end,
1844                         host_page_size, host_huge_range_start,
1845                         host_huge_range_end, host_huge_page_size,
1846                         dram_range_start, dram_range_end, dram_page_size);
1847 }
1848
1849 /*
1850  * hl_vm_ctx_fini       - virtual memory teardown of context
1851  *
1852  * @ctx                 : pointer to the habanalabs context structure
1853  *
1854  * This function perform teardown the following:
1855  * - Virtual block list of available virtual memory
1856  * - Virtual address to area descriptor hashtable
1857  * - MMU for context
1858  *
1859  * In addition this function does the following:
1860  * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
1861  *   hashtable should be empty as no valid mappings should exist at this
1862  *   point.
1863  * - Frees any existing physical page list from the idr which relates to the
1864  *   current context asid.
1865  * - This function checks the virtual block list for correctness. At this point
1866  *   the list should contain one element which describes the whole virtual
1867  *   memory range of the context. Otherwise, a warning is printed.
1868  */
1869 void hl_vm_ctx_fini(struct hl_ctx *ctx)
1870 {
1871         struct hl_device *hdev = ctx->hdev;
1872         struct hl_vm *vm = &hdev->vm;
1873         struct hl_vm_phys_pg_pack *phys_pg_list;
1874         struct hl_vm_hash_node *hnode;
1875         struct hlist_node *tmp_node;
1876         int i;
1877
1878         if (!ctx->hdev->mmu_enable)
1879                 return;
1880
1881         hl_debugfs_remove_ctx_mem_hash(hdev, ctx);
1882
1883         /*
1884          * Clearly something went wrong on hard reset so no point in printing
1885          * another side effect error
1886          */
1887         if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash))
1888                 dev_notice(hdev->dev,
1889                         "user released device without removing its memory mappings\n");
1890
1891         hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) {
1892                 dev_dbg(hdev->dev,
1893                         "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n",
1894                         hnode->vaddr, ctx->asid);
1895                 unmap_device_va(ctx, hnode->vaddr, true);
1896         }
1897
1898         /* invalidate the cache once after the unmapping loop */
1899         hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR);
1900         hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK);
1901
1902         spin_lock(&vm->idr_lock);
1903         idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i)
1904                 if (phys_pg_list->asid == ctx->asid) {
1905                         dev_dbg(hdev->dev,
1906                                 "page list 0x%px of asid %d is still alive\n",
1907                                 phys_pg_list, ctx->asid);
1908                         atomic64_sub(phys_pg_list->total_size,
1909                                         &hdev->dram_used_mem);
1910                         free_phys_pg_pack(hdev, phys_pg_list);
1911                         idr_remove(&vm->phys_pg_pack_handles, i);
1912                 }
1913         spin_unlock(&vm->idr_lock);
1914
1915         va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]);
1916         va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]);
1917
1918         if (hdev->pmmu_huge_range)
1919                 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]);
1920
1921         mutex_destroy(&ctx->mem_hash_lock);
1922         hl_mmu_ctx_fini(ctx);
1923
1924         /* In this case we need to clear the global accounting of DRAM usage
1925          * because the user notifies us on allocations. If the user is no more,
1926          * all DRAM is available
1927          */
1928         if (!ctx->hdev->asic_prop.dram_supports_virtual_memory)
1929                 atomic64_set(&ctx->hdev->dram_used_mem, 0);
1930 }
1931
1932 /*
1933  * hl_vm_init           - initialize virtual memory module
1934  *
1935  * @hdev                : pointer to the habanalabs device structure
1936  *
1937  * This function initializes the following:
1938  * - MMU module
1939  * - DRAM physical pages pool of 2MB
1940  * - Idr for device memory allocation handles
1941  */
1942 int hl_vm_init(struct hl_device *hdev)
1943 {
1944         struct asic_fixed_properties *prop = &hdev->asic_prop;
1945         struct hl_vm *vm = &hdev->vm;
1946         int rc;
1947
1948         vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1);
1949         if (!vm->dram_pg_pool) {
1950                 dev_err(hdev->dev, "Failed to create dram page pool\n");
1951                 return -ENOMEM;
1952         }
1953
1954         kref_init(&vm->dram_pg_pool_refcount);
1955
1956         rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address,
1957                         prop->dram_end_address - prop->dram_user_base_address,
1958                         -1);
1959
1960         if (rc) {
1961                 dev_err(hdev->dev,
1962                         "Failed to add memory to dram page pool %d\n", rc);
1963                 goto pool_add_err;
1964         }
1965
1966         spin_lock_init(&vm->idr_lock);
1967         idr_init(&vm->phys_pg_pack_handles);
1968
1969         atomic64_set(&hdev->dram_used_mem, 0);
1970
1971         vm->init_done = true;
1972
1973         return 0;
1974
1975 pool_add_err:
1976         gen_pool_destroy(vm->dram_pg_pool);
1977
1978         return rc;
1979 }
1980
1981 /*
1982  * hl_vm_fini           - virtual memory module teardown
1983  *
1984  * @hdev                : pointer to the habanalabs device structure
1985  *
1986  * This function perform teardown to the following:
1987  * - Idr for device memory allocation handles
1988  * - DRAM physical pages pool of 2MB
1989  * - MMU module
1990  */
1991 void hl_vm_fini(struct hl_device *hdev)
1992 {
1993         struct hl_vm *vm = &hdev->vm;
1994
1995         if (!vm->init_done)
1996                 return;
1997
1998         /*
1999          * At this point all the contexts should be freed and hence no DRAM
2000          * memory should be in use. Hence the DRAM pool should be freed here.
2001          */
2002         if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1)
2003                 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n",
2004                                 __func__);
2005
2006         vm->init_done = false;
2007 }