drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2014-2018 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23 #include <linux/dma-buf.h>
  24 #include <linux/list.h>
  25 #include <linux/pagemap.h>
  26 #include <linux/sched/mm.h>
  27 #include <linux/sched/task.h>
  28 #include <drm/ttm/ttm_tt.h>
  29
  30 #include <drm/drm_exec.h>
  31
  32 #include "amdgpu_object.h"
  33 #include "amdgpu_gem.h"
  34 #include "amdgpu_vm.h"
  35 #include "amdgpu_hmm.h"
  36 #include "amdgpu_amdkfd.h"
  37 #include "amdgpu_dma_buf.h"
  38 #include <uapi/linux/kfd_ioctl.h>
  39 #include "amdgpu_xgmi.h"
  40 #include "kfd_priv.h"
  41 #include "kfd_smi_events.h"
  42
  43 /* Userptr restore delay, just long enough to allow consecutive VM
  44  * changes to accumulate
  45  */
  46 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
  47 #define AMDGPU_RESERVE_MEM_LIMIT                        (3UL << 29)
  48
  49 /*
  50  * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
  51  * BO chunk
  52  */
  53 #define VRAM_AVAILABLITY_ALIGN (1 << 21)
  54
  55 /* Impose limit on how much memory KFD can use */
  56 static struct {
  57         uint64_t max_system_mem_limit;
  58         uint64_t max_ttm_mem_limit;
  59         int64_t system_mem_used;
  60         int64_t ttm_mem_used;
  61         spinlock_t mem_limit_lock;
  62 } kfd_mem_limit;
  63
  64 static const char * const domain_bit_to_string[] = {
  65                 "CPU",
  66                 "GTT",
  67                 "VRAM",
  68                 "GDS",
  69                 "GWS",
  70                 "OA"
  71 };
  72
  73 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
  74
  75 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
  76
  77 static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
  78                 struct kgd_mem *mem)
  79 {
  80         struct kfd_mem_attachment *entry;
  81
  82         list_for_each_entry(entry, &mem->attachments, list)
  83                 if (entry->bo_va->base.vm == avm)
  84                         return true;
  85
  86         return false;
  87 }
  88
  89 /**
  90  * reuse_dmamap() - Check whether adev can share the original
  91  * userptr BO
  92  *
  93  * If both adev and bo_adev are in direct mapping or
  94  * in the same iommu group, they can share the original BO.
  95  *
  96  * @adev: Device to which can or cannot share the original BO
  97  * @bo_adev: Device to which allocated BO belongs to
  98  *
  99  * Return: returns true if adev can share original userptr BO,
 100  * false otherwise.
 101  */
 102 static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)
 103 {
 104         return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||
 105                         (adev->dev->iommu_group == bo_adev->dev->iommu_group);
 106 }
 107
 108 /* Set memory usage limits. Current, limits are
 109  *  System (TTM + userptr) memory - 15/16th System RAM
 110  *  TTM memory - 3/8th System RAM
 111  */
 112 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
 113 {
 114         struct sysinfo si;
 115         uint64_t mem;
 116
 117         if (kfd_mem_limit.max_system_mem_limit)
 118                 return;
 119
 120         si_meminfo(&si);
 121         mem = si.totalram - si.totalhigh;
 122         mem *= si.mem_unit;
 123
 124         spin_lock_init(&kfd_mem_limit.mem_limit_lock);
 125         kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);
 126         if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT)
 127                 kfd_mem_limit.max_system_mem_limit >>= 1;
 128         else
 129                 kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT;
 130
 131         kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT;
 132         pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
 133                 (kfd_mem_limit.max_system_mem_limit >> 20),
 134                 (kfd_mem_limit.max_ttm_mem_limit >> 20));
 135 }
 136
 137 void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
 138 {
 139         kfd_mem_limit.system_mem_used += size;
 140 }
 141
 142 /* Estimate page table size needed to represent a given memory size
 143  *
 144  * With 4KB pages, we need one 8 byte PTE for each 4KB of memory
 145  * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
 146  * of memory (factor 256K, >> 18). ROCm user mode tries to optimize
 147  * for 2MB pages for TLB efficiency. However, small allocations and
 148  * fragmented system memory still need some 4KB pages. We choose a
 149  * compromise that should work in most cases without reserving too
 150  * much memory for page tables unnecessarily (factor 16K, >> 14).
 151  */
 152
 153 #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)
 154
 155 /**
 156  * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
 157  * of buffer.
 158  *
 159  * @adev: Device to which allocated BO belongs to
 160  * @size: Size of buffer, in bytes, encapsulated by B0. This should be
 161  * equivalent to amdgpu_bo_size(BO)
 162  * @alloc_flag: Flag used in allocating a BO as noted above
 163  * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is
 164  * managed as one compute node in driver for app
 165  *
 166  * Return:
 167  *      returns -ENOMEM in case of error, ZERO otherwise
 168  */
 169 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 170                 uint64_t size, u32 alloc_flag, int8_t xcp_id)
 171 {
 172         uint64_t reserved_for_pt =
 173                 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
 174         size_t system_mem_needed, ttm_mem_needed, vram_needed;
 175         int ret = 0;
 176         uint64_t vram_size = 0;
 177
 178         system_mem_needed = 0;
 179         ttm_mem_needed = 0;
 180         vram_needed = 0;
 181         if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 182                 system_mem_needed = size;
 183                 ttm_mem_needed = size;
 184         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 185                 /*
 186                  * Conservatively round up the allocation requirement to 2 MB
 187                  * to avoid fragmentation caused by 4K allocations in the tail
 188                  * 2M BO chunk.
 189                  */
 190                 vram_needed = size;
 191                 /*
 192                  * For GFX 9.4.3, get the VRAM size from XCP structs
 193                  */
 194                 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
 195                         return -EINVAL;
 196
 197                 vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
 198                 if (adev->gmc.is_app_apu) {
 199                         system_mem_needed = size;
 200                         ttm_mem_needed = size;
 201                 }
 202         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 203                 system_mem_needed = size;
 204         } else if (!(alloc_flag &
 205                                 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
 206                                  KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
 207                 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
 208                 return -ENOMEM;
 209         }
 210
 211         spin_lock(&kfd_mem_limit.mem_limit_lock);
 212
 213         if (kfd_mem_limit.system_mem_used + system_mem_needed >
 214             kfd_mem_limit.max_system_mem_limit)
 215                 pr_debug("Set no_system_mem_limit=1 if using shared memory\n");
 216
 217         if ((kfd_mem_limit.system_mem_used + system_mem_needed >
 218              kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
 219             (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
 220              kfd_mem_limit.max_ttm_mem_limit) ||
 221             (adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >
 222              vram_size - reserved_for_pt)) {
 223                 ret = -ENOMEM;
 224                 goto release;
 225         }
 226
 227         /* Update memory accounting by decreasing available system
 228          * memory, TTM memory and GPU memory as computed above
 229          */
 230         WARN_ONCE(vram_needed && !adev,
 231                   "adev reference can't be null when vram is used");
 232         if (adev && xcp_id >= 0) {
 233                 adev->kfd.vram_used[xcp_id] += vram_needed;
 234                 adev->kfd.vram_used_aligned[xcp_id] += adev->gmc.is_app_apu ?
 235                                 vram_needed :
 236                                 ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
 237         }
 238         kfd_mem_limit.system_mem_used += system_mem_needed;
 239         kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
 240
 241 release:
 242         spin_unlock(&kfd_mem_limit.mem_limit_lock);
 243         return ret;
 244 }
 245
 246 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 247                 uint64_t size, u32 alloc_flag, int8_t xcp_id)
 248 {
 249         spin_lock(&kfd_mem_limit.mem_limit_lock);
 250
 251         if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 252                 kfd_mem_limit.system_mem_used -= size;
 253                 kfd_mem_limit.ttm_mem_used -= size;
 254         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 255                 WARN_ONCE(!adev,
 256                           "adev reference can't be null when alloc mem flags vram is set");
 257                 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
 258                         goto release;
 259
 260                 if (adev) {
 261                         adev->kfd.vram_used[xcp_id] -= size;
 262                         if (adev->gmc.is_app_apu) {
 263                                 adev->kfd.vram_used_aligned[xcp_id] -= size;
 264                                 kfd_mem_limit.system_mem_used -= size;
 265                                 kfd_mem_limit.ttm_mem_used -= size;
 266                         } else {
 267                                 adev->kfd.vram_used_aligned[xcp_id] -=
 268                                         ALIGN(size, VRAM_AVAILABLITY_ALIGN);
 269                         }
 270                 }
 271         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 272                 kfd_mem_limit.system_mem_used -= size;
 273         } else if (!(alloc_flag &
 274                                 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
 275                                  KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
 276                 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
 277                 goto release;
 278         }
 279         WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,
 280                   "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);
 281         WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
 282                   "KFD TTM memory accounting unbalanced");
 283         WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
 284                   "KFD system memory accounting unbalanced");
 285
 286 release:
 287         spin_unlock(&kfd_mem_limit.mem_limit_lock);
 288 }
 289
 290 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
 291 {
 292         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 293         u32 alloc_flags = bo->kfd_bo->alloc_flags;
 294         u64 size = amdgpu_bo_size(bo);
 295
 296         amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags,
 297                                           bo->xcp_id);
 298
 299         kfree(bo->kfd_bo);
 300 }
 301
 302 /**
 303  * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
 304  * about USERPTR or DOOREBELL or MMIO BO.
 305  *
 306  * @adev: Device for which dmamap BO is being created
 307  * @mem: BO of peer device that is being DMA mapped. Provides parameters
 308  *       in building the dmamap BO
 309  * @bo_out: Output parameter updated with handle of dmamap BO
 310  */
 311 static int
 312 create_dmamap_sg_bo(struct amdgpu_device *adev,
 313                  struct kgd_mem *mem, struct amdgpu_bo **bo_out)
 314 {
 315         struct drm_gem_object *gem_obj;
 316         int ret;
 317         uint64_t flags = 0;
 318
 319         ret = amdgpu_bo_reserve(mem->bo, false);
 320         if (ret)
 321                 return ret;
 322
 323         if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
 324                 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
 325                                         AMDGPU_GEM_CREATE_UNCACHED);
 326
 327         ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,
 328                         AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
 329                         ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0);
 330
 331         amdgpu_bo_unreserve(mem->bo);
 332
 333         if (ret) {
 334                 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
 335                 return -EINVAL;
 336         }
 337
 338         *bo_out = gem_to_amdgpu_bo(gem_obj);
 339         (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
 340         return ret;
 341 }
 342
 343 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
 344  *  reservation object.
 345  *
 346  * @bo: [IN] Remove eviction fence(s) from this BO
 347  * @ef: [IN] This eviction fence is removed if it
 348  *  is present in the shared list.
 349  *
 350  * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
 351  */
 352 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
 353                                         struct amdgpu_amdkfd_fence *ef)
 354 {
 355         struct dma_fence *replacement;
 356
 357         if (!ef)
 358                 return -EINVAL;
 359
 360         /* TODO: Instead of block before we should use the fence of the page
 361          * table update and TLB flush here directly.
 362          */
 363         replacement = dma_fence_get_stub();
 364         dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,
 365                                 replacement, DMA_RESV_USAGE_BOOKKEEP);
 366         dma_fence_put(replacement);
 367         return 0;
 368 }
 369
 370 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
 371 {
 372         struct amdgpu_bo *root = bo;
 373         struct amdgpu_vm_bo_base *vm_bo;
 374         struct amdgpu_vm *vm;
 375         struct amdkfd_process_info *info;
 376         struct amdgpu_amdkfd_fence *ef;
 377         int ret;
 378
 379         /* we can always get vm_bo from root PD bo.*/
 380         while (root->parent)
 381                 root = root->parent;
 382
 383         vm_bo = root->vm_bo;
 384         if (!vm_bo)
 385                 return 0;
 386
 387         vm = vm_bo->vm;
 388         if (!vm)
 389                 return 0;
 390
 391         info = vm->process_info;
 392         if (!info || !info->eviction_fence)
 393                 return 0;
 394
 395         ef = container_of(dma_fence_get(&info->eviction_fence->base),
 396                         struct amdgpu_amdkfd_fence, base);
 397
 398         BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));
 399         ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);
 400         dma_resv_unlock(bo->tbo.base.resv);
 401
 402         dma_fence_put(&ef->base);
 403         return ret;
 404 }
 405
 406 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
 407                                      bool wait)
 408 {
 409         struct ttm_operation_ctx ctx = { false, false };
 410         int ret;
 411
 412         if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
 413                  "Called with userptr BO"))
 414                 return -EINVAL;
 415
 416         amdgpu_bo_placement_from_domain(bo, domain);
 417
 418         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 419         if (ret)
 420                 goto validate_fail;
 421         if (wait)
 422                 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
 423
 424 validate_fail:
 425         return ret;
 426 }
 427
 428 static int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
 429                                                uint32_t domain,
 430                                                struct dma_fence *fence)
 431 {
 432         int ret = amdgpu_bo_reserve(bo, false);
 433
 434         if (ret)
 435                 return ret;
 436
 437         ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
 438         if (ret)
 439                 goto unreserve_out;
 440
 441         ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1);
 442         if (ret)
 443                 goto unreserve_out;
 444
 445         dma_resv_add_fence(bo->tbo.base.resv, fence,
 446                            DMA_RESV_USAGE_BOOKKEEP);
 447
 448 unreserve_out:
 449         amdgpu_bo_unreserve(bo);
 450
 451         return ret;
 452 }
 453
 454 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
 455 {
 456         return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);
 457 }
 458
 459 /* vm_validate_pt_pd_bos - Validate page table and directory BOs
 460  *
 461  * Page directories are not updated here because huge page handling
 462  * during page table updates can invalidate page directory entries
 463  * again. Page directories are only updated after updating page
 464  * tables.
 465  */
 466 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
 467 {
 468         struct amdgpu_bo *pd = vm->root.bo;
 469         struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
 470         int ret;
 471
 472         ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate_vm_bo, NULL);
 473         if (ret) {
 474                 pr_err("failed to validate PT BOs\n");
 475                 return ret;
 476         }
 477
 478         vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);
 479
 480         return 0;
 481 }
 482
 483 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
 484 {
 485         struct amdgpu_bo *pd = vm->root.bo;
 486         struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
 487         int ret;
 488
 489         ret = amdgpu_vm_update_pdes(adev, vm, false);
 490         if (ret)
 491                 return ret;
 492
 493         return amdgpu_sync_fence(sync, vm->last_update);
 494 }
 495
 496 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
 497 {
 498         uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |
 499                                  AMDGPU_VM_MTYPE_DEFAULT;
 500
 501         if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
 502                 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
 503         if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
 504                 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
 505
 506         return amdgpu_gem_va_map_flags(adev, mapping_flags);
 507 }
 508
 509 /**
 510  * create_sg_table() - Create an sg_table for a contiguous DMA addr range
 511  * @addr: The starting address to point to
 512  * @size: Size of memory area in bytes being pointed to
 513  *
 514  * Allocates an instance of sg_table and initializes it to point to memory
 515  * area specified by input parameters. The address used to build is assumed
 516  * to be DMA mapped, if needed.
 517  *
 518  * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
 519  * because they are physically contiguous.
 520  *
 521  * Return: Initialized instance of SG Table or NULL
 522  */
 523 static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
 524 {
 525         struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
 526
 527         if (!sg)
 528                 return NULL;
 529         if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
 530                 kfree(sg);
 531                 return NULL;
 532         }
 533         sg_dma_address(sg->sgl) = addr;
 534         sg->sgl->length = size;
 535 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 536         sg->sgl->dma_length = size;
 537 #endif
 538         return sg;
 539 }
 540
 541 static int
 542 kfd_mem_dmamap_userptr(struct kgd_mem *mem,
 543                        struct kfd_mem_attachment *attachment)
 544 {
 545         enum dma_data_direction direction =
 546                 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 547                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 548         struct ttm_operation_ctx ctx = {.interruptible = true};
 549         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 550         struct amdgpu_device *adev = attachment->adev;
 551         struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
 552         struct ttm_tt *ttm = bo->tbo.ttm;
 553         int ret;
 554
 555         if (WARN_ON(ttm->num_pages != src_ttm->num_pages))
 556                 return -EINVAL;
 557
 558         ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL);
 559         if (unlikely(!ttm->sg))
 560                 return -ENOMEM;
 561
 562         /* Same sequence as in amdgpu_ttm_tt_pin_userptr */
 563         ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages,
 564                                         ttm->num_pages, 0,
 565                                         (u64)ttm->num_pages << PAGE_SHIFT,
 566                                         GFP_KERNEL);
 567         if (unlikely(ret))
 568                 goto free_sg;
 569
 570         ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
 571         if (unlikely(ret))
 572                 goto release_sg;
 573
 574         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 575         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 576         if (ret)
 577                 goto unmap_sg;
 578
 579         return 0;
 580
 581 unmap_sg:
 582         dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
 583 release_sg:
 584         pr_err("DMA map userptr failed: %d\n", ret);
 585         sg_free_table(ttm->sg);
 586 free_sg:
 587         kfree(ttm->sg);
 588         ttm->sg = NULL;
 589         return ret;
 590 }
 591
 592 static int
 593 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
 594 {
 595         struct ttm_operation_ctx ctx = {.interruptible = true};
 596         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 597         int ret;
 598
 599         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 600         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 601         if (ret)
 602                 return ret;
 603
 604         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 605         return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 606 }
 607
 608 /**
 609  * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
 610  * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
 611  * @attachment: Virtual address attachment of the BO on accessing device
 612  *
 613  * An access request from the device that owns DOORBELL does not require DMA mapping.
 614  * This is because the request doesn't go through PCIe root complex i.e. it instead
 615  * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
 616  *
 617  * In contrast, all access requests for MMIO need to be DMA mapped without regard to
 618  * device ownership. This is because access requests for MMIO go through PCIe root
 619  * complex.
 620  *
 621  * This is accomplished in two steps:
 622  *   - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
 623  *         in updating requesting device's page table
 624  *   - Signal TTM to mark memory pointed to by requesting device's BO as GPU
 625  *         accessible. This allows an update of requesting device's page table
 626  *         with entries associated with DOOREBELL or MMIO memory
 627  *
 628  * This method is invoked in the following contexts:
 629  *   - Mapping of DOORBELL or MMIO BO of same or peer device
 630  *   - Validating an evicted DOOREBELL or MMIO BO on device seeking access
 631  *
 632  * Return: ZERO if successful, NON-ZERO otherwise
 633  */
 634 static int
 635 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
 636                      struct kfd_mem_attachment *attachment)
 637 {
 638         struct ttm_operation_ctx ctx = {.interruptible = true};
 639         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 640         struct amdgpu_device *adev = attachment->adev;
 641         struct ttm_tt *ttm = bo->tbo.ttm;
 642         enum dma_data_direction dir;
 643         dma_addr_t dma_addr;
 644         bool mmio;
 645         int ret;
 646
 647         /* Expect SG Table of dmapmap BO to be NULL */
 648         mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
 649         if (unlikely(ttm->sg)) {
 650                 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
 651                 return -EINVAL;
 652         }
 653
 654         dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 655                         DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 656         dma_addr = mem->bo->tbo.sg->sgl->dma_address;
 657         pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
 658         pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
 659         dma_addr = dma_map_resource(adev->dev, dma_addr,
 660                         mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
 661         ret = dma_mapping_error(adev->dev, dma_addr);
 662         if (unlikely(ret))
 663                 return ret;
 664         pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
 665
 666         ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
 667         if (unlikely(!ttm->sg)) {
 668                 ret = -ENOMEM;
 669                 goto unmap_sg;
 670         }
 671
 672         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 673         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 674         if (unlikely(ret))
 675                 goto free_sg;
 676
 677         return ret;
 678
 679 free_sg:
 680         sg_free_table(ttm->sg);
 681         kfree(ttm->sg);
 682         ttm->sg = NULL;
 683 unmap_sg:
 684         dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
 685                            dir, DMA_ATTR_SKIP_CPU_SYNC);
 686         return ret;
 687 }
 688
 689 static int
 690 kfd_mem_dmamap_attachment(struct kgd_mem *mem,
 691                           struct kfd_mem_attachment *attachment)
 692 {
 693         switch (attachment->type) {
 694         case KFD_MEM_ATT_SHARED:
 695                 return 0;
 696         case KFD_MEM_ATT_USERPTR:
 697                 return kfd_mem_dmamap_userptr(mem, attachment);
 698         case KFD_MEM_ATT_DMABUF:
 699                 return kfd_mem_dmamap_dmabuf(attachment);
 700         case KFD_MEM_ATT_SG:
 701                 return kfd_mem_dmamap_sg_bo(mem, attachment);
 702         default:
 703                 WARN_ON_ONCE(1);
 704         }
 705         return -EINVAL;
 706 }
 707
 708 static void
 709 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
 710                          struct kfd_mem_attachment *attachment)
 711 {
 712         enum dma_data_direction direction =
 713                 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 714                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 715         struct ttm_operation_ctx ctx = {.interruptible = false};
 716         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 717         struct amdgpu_device *adev = attachment->adev;
 718         struct ttm_tt *ttm = bo->tbo.ttm;
 719
 720         if (unlikely(!ttm->sg))
 721                 return;
 722
 723         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 724         ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 725
 726         dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
 727         sg_free_table(ttm->sg);
 728         kfree(ttm->sg);
 729         ttm->sg = NULL;
 730 }
 731
 732 static void
 733 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
 734 {
 735         /* This is a no-op. We don't want to trigger eviction fences when
 736          * unmapping DMABufs. Therefore the invalidation (moving to system
 737          * domain) is done in kfd_mem_dmamap_dmabuf.
 738          */
 739 }
 740
 741 /**
 742  * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
 743  * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
 744  * @attachment: Virtual address attachment of the BO on accessing device
 745  *
 746  * The method performs following steps:
 747  *   - Signal TTM to mark memory pointed to by BO as GPU inaccessible
 748  *   - Free SG Table that is used to encapsulate DMA mapped memory of
 749  *          peer device's DOORBELL or MMIO memory
 750  *
 751  * This method is invoked in the following contexts:
 752  *     UNMapping of DOORBELL or MMIO BO on a device having access to its memory
 753  *     Eviction of DOOREBELL or MMIO BO on device having access to its memory
 754  *
 755  * Return: void
 756  */
 757 static void
 758 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
 759                        struct kfd_mem_attachment *attachment)
 760 {
 761         struct ttm_operation_ctx ctx = {.interruptible = true};
 762         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 763         struct amdgpu_device *adev = attachment->adev;
 764         struct ttm_tt *ttm = bo->tbo.ttm;
 765         enum dma_data_direction dir;
 766
 767         if (unlikely(!ttm->sg)) {
 768                 pr_debug("SG Table of BO is NULL");
 769                 return;
 770         }
 771
 772         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 773         ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 774
 775         dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 776                                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 777         dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
 778                         ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
 779         sg_free_table(ttm->sg);
 780         kfree(ttm->sg);
 781         ttm->sg = NULL;
 782         bo->tbo.sg = NULL;
 783 }
 784
 785 static void
 786 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
 787                             struct kfd_mem_attachment *attachment)
 788 {
 789         switch (attachment->type) {
 790         case KFD_MEM_ATT_SHARED:
 791                 break;
 792         case KFD_MEM_ATT_USERPTR:
 793                 kfd_mem_dmaunmap_userptr(mem, attachment);
 794                 break;
 795         case KFD_MEM_ATT_DMABUF:
 796                 kfd_mem_dmaunmap_dmabuf(attachment);
 797                 break;
 798         case KFD_MEM_ATT_SG:
 799                 kfd_mem_dmaunmap_sg_bo(mem, attachment);
 800                 break;
 801         default:
 802                 WARN_ON_ONCE(1);
 803         }
 804 }
 805
 806 static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
 807 {
 808         if (!mem->dmabuf) {
 809                 struct dma_buf *ret = amdgpu_gem_prime_export(
 810                         &mem->bo->tbo.base,
 811                         mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 812                                 DRM_RDWR : 0);
 813                 if (IS_ERR(ret))
 814                         return PTR_ERR(ret);
 815                 mem->dmabuf = ret;
 816         }
 817
 818         return 0;
 819 }
 820
 821 static int
 822 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
 823                       struct amdgpu_bo **bo)
 824 {
 825         struct drm_gem_object *gobj;
 826         int ret;
 827
 828         ret = kfd_mem_export_dmabuf(mem);
 829         if (ret)
 830                 return ret;
 831
 832         gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf);
 833         if (IS_ERR(gobj))
 834                 return PTR_ERR(gobj);
 835
 836         *bo = gem_to_amdgpu_bo(gobj);
 837         (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
 838
 839         return 0;
 840 }
 841
 842 /* kfd_mem_attach - Add a BO to a VM
 843  *
 844  * Everything that needs to bo done only once when a BO is first added
 845  * to a VM. It can later be mapped and unmapped many times without
 846  * repeating these steps.
 847  *
 848  * 0. Create BO for DMA mapping, if needed
 849  * 1. Allocate and initialize BO VA entry data structure
 850  * 2. Add BO to the VM
 851  * 3. Determine ASIC-specific PTE flags
 852  * 4. Alloc page tables and directories if needed
 853  * 4a.  Validate new page tables and directories
 854  */
 855 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
 856                 struct amdgpu_vm *vm, bool is_aql)
 857 {
 858         struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
 859         unsigned long bo_size = mem->bo->tbo.base.size;
 860         uint64_t va = mem->va;
 861         struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
 862         struct amdgpu_bo *bo[2] = {NULL, NULL};
 863         struct amdgpu_bo_va *bo_va;
 864         bool same_hive = false;
 865         int i, ret;
 866
 867         if (!va) {
 868                 pr_err("Invalid VA when adding BO to VM\n");
 869                 return -EINVAL;
 870         }
 871
 872         /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
 873          *
 874          * The access path of MMIO and DOORBELL BOs of is always over PCIe.
 875          * In contrast the access path of VRAM BOs depens upon the type of
 876          * link that connects the peer device. Access over PCIe is allowed
 877          * if peer device has large BAR. In contrast, access over xGMI is
 878          * allowed for both small and large BAR configurations of peer device
 879          */
 880         if ((adev != bo_adev && !adev->gmc.is_app_apu) &&
 881             ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
 882              (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
 883              (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
 884                 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
 885                         same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
 886                 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
 887                         return -EINVAL;
 888         }
 889
 890         for (i = 0; i <= is_aql; i++) {
 891                 attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
 892                 if (unlikely(!attachment[i])) {
 893                         ret = -ENOMEM;
 894                         goto unwind;
 895                 }
 896
 897                 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
 898                          va + bo_size, vm);
 899
 900                 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
 901                     (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
 902                     (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||
 903                     same_hive) {
 904                         /* Mappings on the local GPU, or VRAM mappings in the
 905                          * local hive, or userptr, or GTT mapping can reuse dma map
 906                          * address space share the original BO
 907                          */
 908                         attachment[i]->type = KFD_MEM_ATT_SHARED;
 909                         bo[i] = mem->bo;
 910                         drm_gem_object_get(&bo[i]->tbo.base);
 911                 } else if (i > 0) {
 912                         /* Multiple mappings on the same GPU share the BO */
 913                         attachment[i]->type = KFD_MEM_ATT_SHARED;
 914                         bo[i] = bo[0];
 915                         drm_gem_object_get(&bo[i]->tbo.base);
 916                 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
 917                         /* Create an SG BO to DMA-map userptrs on other GPUs */
 918                         attachment[i]->type = KFD_MEM_ATT_USERPTR;
 919                         ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
 920                         if (ret)
 921                                 goto unwind;
 922                 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
 923                 } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
 924                         WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
 925                                     mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
 926                                   "Handing invalid SG BO in ATTACH request");
 927                         attachment[i]->type = KFD_MEM_ATT_SG;
 928                         ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
 929                         if (ret)
 930                                 goto unwind;
 931                 /* Enable acces to GTT and VRAM BOs of peer devices */
 932                 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
 933                            mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
 934                         attachment[i]->type = KFD_MEM_ATT_DMABUF;
 935                         ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
 936                         if (ret)
 937                                 goto unwind;
 938                         pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");
 939                 } else {
 940                         WARN_ONCE(true, "Handling invalid ATTACH request");
 941                         ret = -EINVAL;
 942                         goto unwind;
 943                 }
 944
 945                 /* Add BO to VM internal data structures */
 946                 ret = amdgpu_bo_reserve(bo[i], false);
 947                 if (ret) {
 948                         pr_debug("Unable to reserve BO during memory attach");
 949                         goto unwind;
 950                 }
 951                 bo_va = amdgpu_vm_bo_find(vm, bo[i]);
 952                 if (!bo_va)
 953                         bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
 954                 else
 955                         ++bo_va->ref_count;
 956                 attachment[i]->bo_va = bo_va;
 957                 amdgpu_bo_unreserve(bo[i]);
 958                 if (unlikely(!attachment[i]->bo_va)) {
 959                         ret = -ENOMEM;
 960                         pr_err("Failed to add BO object to VM. ret == %d\n",
 961                                ret);
 962                         goto unwind;
 963                 }
 964                 attachment[i]->va = va;
 965                 attachment[i]->pte_flags = get_pte_flags(adev, mem);
 966                 attachment[i]->adev = adev;
 967                 list_add(&attachment[i]->list, &mem->attachments);
 968
 969                 va += bo_size;
 970         }
 971
 972         return 0;
 973
 974 unwind:
 975         for (; i >= 0; i--) {
 976                 if (!attachment[i])
 977                         continue;
 978                 if (attachment[i]->bo_va) {
 979                         amdgpu_bo_reserve(bo[i], true);
 980                         if (--attachment[i]->bo_va->ref_count == 0)
 981                                 amdgpu_vm_bo_del(adev, attachment[i]->bo_va);
 982                         amdgpu_bo_unreserve(bo[i]);
 983                         list_del(&attachment[i]->list);
 984                 }
 985                 if (bo[i])
 986                         drm_gem_object_put(&bo[i]->tbo.base);
 987                 kfree(attachment[i]);
 988         }
 989         return ret;
 990 }
 991
 992 static void kfd_mem_detach(struct kfd_mem_attachment *attachment)
 993 {
 994         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 995
 996         pr_debug("\t remove VA 0x%llx in entry %p\n",
 997                         attachment->va, attachment);
 998         if (--attachment->bo_va->ref_count == 0)
 999                 amdgpu_vm_bo_del(attachment->adev, attachment->bo_va);
1000         drm_gem_object_put(&bo->tbo.base);
1001         list_del(&attachment->list);
1002         kfree(attachment);
1003 }
1004
1005 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
1006                                 struct amdkfd_process_info *process_info,
1007                                 bool userptr)
1008 {
1009         mutex_lock(&process_info->lock);
1010         if (userptr)
1011                 list_add_tail(&mem->validate_list,
1012                               &process_info->userptr_valid_list);
1013         else
1014                 list_add_tail(&mem->validate_list, &process_info->kfd_bo_list);
1015         mutex_unlock(&process_info->lock);
1016 }
1017
1018 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
1019                 struct amdkfd_process_info *process_info)
1020 {
1021         mutex_lock(&process_info->lock);
1022         list_del(&mem->validate_list);
1023         mutex_unlock(&process_info->lock);
1024 }
1025
1026 /* Initializes user pages. It registers the MMU notifier and validates
1027  * the userptr BO in the GTT domain.
1028  *
1029  * The BO must already be on the userptr_valid_list. Otherwise an
1030  * eviction and restore may happen that leaves the new BO unmapped
1031  * with the user mode queues running.
1032  *
1033  * Takes the process_info->lock to protect against concurrent restore
1034  * workers.
1035  *
1036  * Returns 0 for success, negative errno for errors.
1037  */
1038 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
1039                            bool criu_resume)
1040 {
1041         struct amdkfd_process_info *process_info = mem->process_info;
1042         struct amdgpu_bo *bo = mem->bo;
1043         struct ttm_operation_ctx ctx = { true, false };
1044         struct hmm_range *range;
1045         int ret = 0;
1046
1047         mutex_lock(&process_info->lock);
1048
1049         ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);
1050         if (ret) {
1051                 pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
1052                 goto out;
1053         }
1054
1055         ret = amdgpu_hmm_register(bo, user_addr);
1056         if (ret) {
1057                 pr_err("%s: Failed to register MMU notifier: %d\n",
1058                        __func__, ret);
1059                 goto out;
1060         }
1061
1062         if (criu_resume) {
1063                 /*
1064                  * During a CRIU restore operation, the userptr buffer objects
1065                  * will be validated in the restore_userptr_work worker at a
1066                  * later stage when it is scheduled by another ioctl called by
1067                  * CRIU master process for the target pid for restore.
1068                  */
1069                 mutex_lock(&process_info->notifier_lock);
1070                 mem->invalid++;
1071                 mutex_unlock(&process_info->notifier_lock);
1072                 mutex_unlock(&process_info->lock);
1073                 return 0;
1074         }
1075
1076         ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range);
1077         if (ret) {
1078                 pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
1079                 goto unregister_out;
1080         }
1081
1082         ret = amdgpu_bo_reserve(bo, true);
1083         if (ret) {
1084                 pr_err("%s: Failed to reserve BO\n", __func__);
1085                 goto release_out;
1086         }
1087         amdgpu_bo_placement_from_domain(bo, mem->domain);
1088         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1089         if (ret)
1090                 pr_err("%s: failed to validate BO\n", __func__);
1091         amdgpu_bo_unreserve(bo);
1092
1093 release_out:
1094         amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);
1095 unregister_out:
1096         if (ret)
1097                 amdgpu_hmm_unregister(bo);
1098 out:
1099         mutex_unlock(&process_info->lock);
1100         return ret;
1101 }
1102
1103 /* Reserving a BO and its page table BOs must happen atomically to
1104  * avoid deadlocks. Some operations update multiple VMs at once. Track
1105  * all the reservation info in a context structure. Optionally a sync
1106  * object can track VM updates.
1107  */
1108 struct bo_vm_reservation_context {
1109         /* DRM execution context for the reservation */
1110         struct drm_exec exec;
1111         /* Number of VMs reserved */
1112         unsigned int n_vms;
1113         /* Pointer to sync object */
1114         struct amdgpu_sync *sync;
1115 };
1116
1117 enum bo_vm_match {
1118         BO_VM_NOT_MAPPED = 0,   /* Match VMs where a BO is not mapped */
1119         BO_VM_MAPPED,           /* Match VMs where a BO is mapped     */
1120         BO_VM_ALL,              /* Match all VMs a BO was added to    */
1121 };
1122
1123 /**
1124  * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
1125  * @mem: KFD BO structure.
1126  * @vm: the VM to reserve.
1127  * @ctx: the struct that will be used in unreserve_bo_and_vms().
1128  */
1129 static int reserve_bo_and_vm(struct kgd_mem *mem,
1130                               struct amdgpu_vm *vm,
1131                               struct bo_vm_reservation_context *ctx)
1132 {
1133         struct amdgpu_bo *bo = mem->bo;
1134         int ret;
1135
1136         WARN_ON(!vm);
1137
1138         ctx->n_vms = 1;
1139         ctx->sync = &mem->sync;
1140         drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
1141         drm_exec_until_all_locked(&ctx->exec) {
1142                 ret = amdgpu_vm_lock_pd(vm, &ctx->exec, 2);
1143                 drm_exec_retry_on_contention(&ctx->exec);
1144                 if (unlikely(ret))
1145                         goto error;
1146
1147                 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1);
1148                 drm_exec_retry_on_contention(&ctx->exec);
1149                 if (unlikely(ret))
1150                         goto error;
1151         }
1152         return 0;
1153
1154 error:
1155         pr_err("Failed to reserve buffers in ttm.\n");
1156         drm_exec_fini(&ctx->exec);
1157         return ret;
1158 }
1159
1160 /**
1161  * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
1162  * @mem: KFD BO structure.
1163  * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
1164  * is used. Otherwise, a single VM associated with the BO.
1165  * @map_type: the mapping status that will be used to filter the VMs.
1166  * @ctx: the struct that will be used in unreserve_bo_and_vms().
1167  *
1168  * Returns 0 for success, negative for failure.
1169  */
1170 static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1171                                 struct amdgpu_vm *vm, enum bo_vm_match map_type,
1172                                 struct bo_vm_reservation_context *ctx)
1173 {
1174         struct kfd_mem_attachment *entry;
1175         struct amdgpu_bo *bo = mem->bo;
1176         int ret;
1177
1178         ctx->sync = &mem->sync;
1179         drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
1180         drm_exec_until_all_locked(&ctx->exec) {
1181                 ctx->n_vms = 0;
1182                 list_for_each_entry(entry, &mem->attachments, list) {
1183                         if ((vm && vm != entry->bo_va->base.vm) ||
1184                                 (entry->is_mapped != map_type
1185                                 && map_type != BO_VM_ALL))
1186                                 continue;
1187
1188                         ret = amdgpu_vm_lock_pd(entry->bo_va->base.vm,
1189                                                 &ctx->exec, 2);
1190                         drm_exec_retry_on_contention(&ctx->exec);
1191                         if (unlikely(ret))
1192                                 goto error;
1193                         ++ctx->n_vms;
1194                 }
1195
1196                 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1);
1197                 drm_exec_retry_on_contention(&ctx->exec);
1198                 if (unlikely(ret))
1199                         goto error;
1200         }
1201         return 0;
1202
1203 error:
1204         pr_err("Failed to reserve buffers in ttm.\n");
1205         drm_exec_fini(&ctx->exec);
1206         return ret;
1207 }
1208
1209 /**
1210  * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
1211  * @ctx: Reservation context to unreserve
1212  * @wait: Optionally wait for a sync object representing pending VM updates
1213  * @intr: Whether the wait is interruptible
1214  *
1215  * Also frees any resources allocated in
1216  * reserve_bo_and_(cond_)vm(s). Returns the status from
1217  * amdgpu_sync_wait.
1218  */
1219 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
1220                                  bool wait, bool intr)
1221 {
1222         int ret = 0;
1223
1224         if (wait)
1225                 ret = amdgpu_sync_wait(ctx->sync, intr);
1226
1227         drm_exec_fini(&ctx->exec);
1228         ctx->sync = NULL;
1229         return ret;
1230 }
1231
1232 static void unmap_bo_from_gpuvm(struct kgd_mem *mem,
1233                                 struct kfd_mem_attachment *entry,
1234                                 struct amdgpu_sync *sync)
1235 {
1236         struct amdgpu_bo_va *bo_va = entry->bo_va;
1237         struct amdgpu_device *adev = entry->adev;
1238         struct amdgpu_vm *vm = bo_va->base.vm;
1239
1240         amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
1241
1242         amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
1243
1244         amdgpu_sync_fence(sync, bo_va->last_pt_update);
1245 }
1246
1247 static int update_gpuvm_pte(struct kgd_mem *mem,
1248                             struct kfd_mem_attachment *entry,
1249                             struct amdgpu_sync *sync)
1250 {
1251         struct amdgpu_bo_va *bo_va = entry->bo_va;
1252         struct amdgpu_device *adev = entry->adev;
1253         int ret;
1254
1255         ret = kfd_mem_dmamap_attachment(mem, entry);
1256         if (ret)
1257                 return ret;
1258
1259         /* Update the page tables  */
1260         ret = amdgpu_vm_bo_update(adev, bo_va, false);
1261         if (ret) {
1262                 pr_err("amdgpu_vm_bo_update failed\n");
1263                 return ret;
1264         }
1265
1266         return amdgpu_sync_fence(sync, bo_va->last_pt_update);
1267 }
1268
1269 static int map_bo_to_gpuvm(struct kgd_mem *mem,
1270                            struct kfd_mem_attachment *entry,
1271                            struct amdgpu_sync *sync,
1272                            bool no_update_pte)
1273 {
1274         int ret;
1275
1276         /* Set virtual address for the allocation */
1277         ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0,
1278                                amdgpu_bo_size(entry->bo_va->base.bo),
1279                                entry->pte_flags);
1280         if (ret) {
1281                 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
1282                                 entry->va, ret);
1283                 return ret;
1284         }
1285
1286         if (no_update_pte)
1287                 return 0;
1288
1289         ret = update_gpuvm_pte(mem, entry, sync);
1290         if (ret) {
1291                 pr_err("update_gpuvm_pte() failed\n");
1292                 goto update_gpuvm_pte_failed;
1293         }
1294
1295         return 0;
1296
1297 update_gpuvm_pte_failed:
1298         unmap_bo_from_gpuvm(mem, entry, sync);
1299         kfd_mem_dmaunmap_attachment(mem, entry);
1300         return ret;
1301 }
1302
1303 static int process_validate_vms(struct amdkfd_process_info *process_info)
1304 {
1305         struct amdgpu_vm *peer_vm;
1306         int ret;
1307
1308         list_for_each_entry(peer_vm, &process_info->vm_list_head,
1309                             vm_list_node) {
1310                 ret = vm_validate_pt_pd_bos(peer_vm);
1311                 if (ret)
1312                         return ret;
1313         }
1314
1315         return 0;
1316 }
1317
1318 static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
1319                                  struct amdgpu_sync *sync)
1320 {
1321         struct amdgpu_vm *peer_vm;
1322         int ret;
1323
1324         list_for_each_entry(peer_vm, &process_info->vm_list_head,
1325                             vm_list_node) {
1326                 struct amdgpu_bo *pd = peer_vm->root.bo;
1327
1328                 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv,
1329                                        AMDGPU_SYNC_NE_OWNER,
1330                                        AMDGPU_FENCE_OWNER_KFD);
1331                 if (ret)
1332                         return ret;
1333         }
1334
1335         return 0;
1336 }
1337
1338 static int process_update_pds(struct amdkfd_process_info *process_info,
1339                               struct amdgpu_sync *sync)
1340 {
1341         struct amdgpu_vm *peer_vm;
1342         int ret;
1343
1344         list_for_each_entry(peer_vm, &process_info->vm_list_head,
1345                             vm_list_node) {
1346                 ret = vm_update_pds(peer_vm, sync);
1347                 if (ret)
1348                         return ret;
1349         }
1350
1351         return 0;
1352 }
1353
1354 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
1355                        struct dma_fence **ef)
1356 {
1357         struct amdkfd_process_info *info = NULL;
1358         int ret;
1359
1360         if (!*process_info) {
1361                 info = kzalloc(sizeof(*info), GFP_KERNEL);
1362                 if (!info)
1363                         return -ENOMEM;
1364
1365                 mutex_init(&info->lock);
1366                 mutex_init(&info->notifier_lock);
1367                 INIT_LIST_HEAD(&info->vm_list_head);
1368                 INIT_LIST_HEAD(&info->kfd_bo_list);
1369                 INIT_LIST_HEAD(&info->userptr_valid_list);
1370                 INIT_LIST_HEAD(&info->userptr_inval_list);
1371
1372                 info->eviction_fence =
1373                         amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
1374                                                    current->mm,
1375                                                    NULL);
1376                 if (!info->eviction_fence) {
1377                         pr_err("Failed to create eviction fence\n");
1378                         ret = -ENOMEM;
1379                         goto create_evict_fence_fail;
1380                 }
1381
1382                 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);
1383                 INIT_DELAYED_WORK(&info->restore_userptr_work,
1384                                   amdgpu_amdkfd_restore_userptr_worker);
1385
1386                 *process_info = info;
1387                 *ef = dma_fence_get(&info->eviction_fence->base);
1388         }
1389
1390         vm->process_info = *process_info;
1391
1392         /* Validate page directory and attach eviction fence */
1393         ret = amdgpu_bo_reserve(vm->root.bo, true);
1394         if (ret)
1395                 goto reserve_pd_fail;
1396         ret = vm_validate_pt_pd_bos(vm);
1397         if (ret) {
1398                 pr_err("validate_pt_pd_bos() failed\n");
1399                 goto validate_pd_fail;
1400         }
1401         ret = amdgpu_bo_sync_wait(vm->root.bo,
1402                                   AMDGPU_FENCE_OWNER_KFD, false);
1403         if (ret)
1404                 goto wait_pd_fail;
1405         ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
1406         if (ret)
1407                 goto reserve_shared_fail;
1408         dma_resv_add_fence(vm->root.bo->tbo.base.resv,
1409                            &vm->process_info->eviction_fence->base,
1410                            DMA_RESV_USAGE_BOOKKEEP);
1411         amdgpu_bo_unreserve(vm->root.bo);
1412
1413         /* Update process info */
1414         mutex_lock(&vm->process_info->lock);
1415         list_add_tail(&vm->vm_list_node,
1416                         &(vm->process_info->vm_list_head));
1417         vm->process_info->n_vms++;
1418         mutex_unlock(&vm->process_info->lock);
1419
1420         return 0;
1421
1422 reserve_shared_fail:
1423 wait_pd_fail:
1424 validate_pd_fail:
1425         amdgpu_bo_unreserve(vm->root.bo);
1426 reserve_pd_fail:
1427         vm->process_info = NULL;
1428         if (info) {
1429                 /* Two fence references: one in info and one in *ef */
1430                 dma_fence_put(&info->eviction_fence->base);
1431                 dma_fence_put(*ef);
1432                 *ef = NULL;
1433                 *process_info = NULL;
1434                 put_pid(info->pid);
1435 create_evict_fence_fail:
1436                 mutex_destroy(&info->lock);
1437                 mutex_destroy(&info->notifier_lock);
1438                 kfree(info);
1439         }
1440         return ret;
1441 }
1442
1443 /**
1444  * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
1445  * @bo: Handle of buffer object being pinned
1446  * @domain: Domain into which BO should be pinned
1447  *
1448  *   - USERPTR BOs are UNPINNABLE and will return error
1449  *   - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1450  *     PIN count incremented. It is valid to PIN a BO multiple times
1451  *
1452  * Return: ZERO if successful in pinning, Non-Zero in case of error.
1453  */
1454 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
1455 {
1456         int ret = 0;
1457
1458         ret = amdgpu_bo_reserve(bo, false);
1459         if (unlikely(ret))
1460                 return ret;
1461
1462         ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
1463         if (ret)
1464                 pr_err("Error in Pinning BO to domain: %d\n", domain);
1465
1466         amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
1467         amdgpu_bo_unreserve(bo);
1468
1469         return ret;
1470 }
1471
1472 /**
1473  * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
1474  * @bo: Handle of buffer object being unpinned
1475  *
1476  *   - Is a illegal request for USERPTR BOs and is ignored
1477  *   - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1478  *     PIN count decremented. Calls to UNPIN must balance calls to PIN
1479  */
1480 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
1481 {
1482         int ret = 0;
1483
1484         ret = amdgpu_bo_reserve(bo, false);
1485         if (unlikely(ret))
1486                 return;
1487
1488         amdgpu_bo_unpin(bo);
1489         amdgpu_bo_unreserve(bo);
1490 }
1491
1492 int amdgpu_amdkfd_gpuvm_set_vm_pasid(struct amdgpu_device *adev,
1493                                      struct amdgpu_vm *avm, u32 pasid)
1494
1495 {
1496         int ret;
1497
1498         /* Free the original amdgpu allocated pasid,
1499          * will be replaced with kfd allocated pasid.
1500          */
1501         if (avm->pasid) {
1502                 amdgpu_pasid_free(avm->pasid);
1503                 amdgpu_vm_set_pasid(adev, avm, 0);
1504         }
1505
1506         ret = amdgpu_vm_set_pasid(adev, avm, pasid);
1507         if (ret)
1508                 return ret;
1509
1510         return 0;
1511 }
1512
1513 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
1514                                            struct amdgpu_vm *avm,
1515                                            void **process_info,
1516                                            struct dma_fence **ef)
1517 {
1518         int ret;
1519
1520         /* Already a compute VM? */
1521         if (avm->process_info)
1522                 return -EINVAL;
1523
1524         /* Convert VM into a compute VM */
1525         ret = amdgpu_vm_make_compute(adev, avm);
1526         if (ret)
1527                 return ret;
1528
1529         /* Initialize KFD part of the VM and process info */
1530         ret = init_kfd_vm(avm, process_info, ef);
1531         if (ret)
1532                 return ret;
1533
1534         amdgpu_vm_set_task_info(avm);
1535
1536         return 0;
1537 }
1538
1539 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
1540                                     struct amdgpu_vm *vm)
1541 {
1542         struct amdkfd_process_info *process_info = vm->process_info;
1543
1544         if (!process_info)
1545                 return;
1546
1547         /* Update process info */
1548         mutex_lock(&process_info->lock);
1549         process_info->n_vms--;
1550         list_del(&vm->vm_list_node);
1551         mutex_unlock(&process_info->lock);
1552
1553         vm->process_info = NULL;
1554
1555         /* Release per-process resources when last compute VM is destroyed */
1556         if (!process_info->n_vms) {
1557                 WARN_ON(!list_empty(&process_info->kfd_bo_list));
1558                 WARN_ON(!list_empty(&process_info->userptr_valid_list));
1559                 WARN_ON(!list_empty(&process_info->userptr_inval_list));
1560
1561                 dma_fence_put(&process_info->eviction_fence->base);
1562                 cancel_delayed_work_sync(&process_info->restore_userptr_work);
1563                 put_pid(process_info->pid);
1564                 mutex_destroy(&process_info->lock);
1565                 mutex_destroy(&process_info->notifier_lock);
1566                 kfree(process_info);
1567         }
1568 }
1569
1570 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
1571                                             void *drm_priv)
1572 {
1573         struct amdgpu_vm *avm;
1574
1575         if (WARN_ON(!adev || !drm_priv))
1576                 return;
1577
1578         avm = drm_priv_to_vm(drm_priv);
1579
1580         pr_debug("Releasing process vm %p\n", avm);
1581
1582         /* The original pasid of amdgpu vm has already been
1583          * released during making a amdgpu vm to a compute vm
1584          * The current pasid is managed by kfd and will be
1585          * released on kfd process destroy. Set amdgpu pasid
1586          * to 0 to avoid duplicate release.
1587          */
1588         amdgpu_vm_release_compute(adev, avm);
1589 }
1590
1591 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1592 {
1593         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1594         struct amdgpu_bo *pd = avm->root.bo;
1595         struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
1596
1597         if (adev->asic_type < CHIP_VEGA10)
1598                 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1599         return avm->pd_phys_addr;
1600 }
1601
1602 void amdgpu_amdkfd_block_mmu_notifications(void *p)
1603 {
1604         struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1605
1606         mutex_lock(&pinfo->lock);
1607         WRITE_ONCE(pinfo->block_mmu_notifications, true);
1608         mutex_unlock(&pinfo->lock);
1609 }
1610
1611 int amdgpu_amdkfd_criu_resume(void *p)
1612 {
1613         int ret = 0;
1614         struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1615
1616         mutex_lock(&pinfo->lock);
1617         pr_debug("scheduling work\n");
1618         mutex_lock(&pinfo->notifier_lock);
1619         pinfo->evicted_bos++;
1620         mutex_unlock(&pinfo->notifier_lock);
1621         if (!READ_ONCE(pinfo->block_mmu_notifications)) {
1622                 ret = -EINVAL;
1623                 goto out_unlock;
1624         }
1625         WRITE_ONCE(pinfo->block_mmu_notifications, false);
1626         schedule_delayed_work(&pinfo->restore_userptr_work, 0);
1627
1628 out_unlock:
1629         mutex_unlock(&pinfo->lock);
1630         return ret;
1631 }
1632
1633 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
1634                                           uint8_t xcp_id)
1635 {
1636         uint64_t reserved_for_pt =
1637                 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
1638         ssize_t available;
1639         uint64_t vram_available, system_mem_available, ttm_mem_available;
1640
1641         spin_lock(&kfd_mem_limit.mem_limit_lock);
1642         vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
1643                 - adev->kfd.vram_used_aligned[xcp_id]
1644                 - atomic64_read(&adev->vram_pin_size)
1645                 - reserved_for_pt;
1646
1647         if (adev->gmc.is_app_apu) {
1648                 system_mem_available = no_system_mem_limit ?
1649                                         kfd_mem_limit.max_system_mem_limit :
1650                                         kfd_mem_limit.max_system_mem_limit -
1651                                         kfd_mem_limit.system_mem_used;
1652
1653                 ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -
1654                                 kfd_mem_limit.ttm_mem_used;
1655
1656                 available = min3(system_mem_available, ttm_mem_available,
1657                                  vram_available);
1658                 available = ALIGN_DOWN(available, PAGE_SIZE);
1659         } else {
1660                 available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
1661         }
1662
1663         spin_unlock(&kfd_mem_limit.mem_limit_lock);
1664
1665         if (available < 0)
1666                 available = 0;
1667
1668         return available;
1669 }
1670
1671 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1672                 struct amdgpu_device *adev, uint64_t va, uint64_t size,
1673                 void *drm_priv, struct kgd_mem **mem,
1674                 uint64_t *offset, uint32_t flags, bool criu_resume)
1675 {
1676         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1677         struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
1678         enum ttm_bo_type bo_type = ttm_bo_type_device;
1679         struct sg_table *sg = NULL;
1680         uint64_t user_addr = 0;
1681         struct amdgpu_bo *bo;
1682         struct drm_gem_object *gobj = NULL;
1683         u32 domain, alloc_domain;
1684         uint64_t aligned_size;
1685         int8_t xcp_id = -1;
1686         u64 alloc_flags;
1687         int ret;
1688
1689         /*
1690          * Check on which domain to allocate BO
1691          */
1692         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1693                 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1694
1695                 if (adev->gmc.is_app_apu) {
1696                         domain = AMDGPU_GEM_DOMAIN_GTT;
1697                         alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1698                         alloc_flags = 0;
1699                 } else {
1700                         alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1701                         alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
1702                         AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
1703                 }
1704                 xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?
1705                                         0 : fpriv->xcp_id;
1706         } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
1707                 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1708                 alloc_flags = 0;
1709         } else {
1710                 domain = AMDGPU_GEM_DOMAIN_GTT;
1711                 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1712                 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
1713
1714                 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1715                         if (!offset || !*offset)
1716                                 return -EINVAL;
1717                         user_addr = untagged_addr(*offset);
1718                 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1719                                     KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1720                         bo_type = ttm_bo_type_sg;
1721                         if (size > UINT_MAX)
1722                                 return -EINVAL;
1723                         sg = create_sg_table(*offset, size);
1724                         if (!sg)
1725                                 return -ENOMEM;
1726                 } else {
1727                         return -EINVAL;
1728                 }
1729         }
1730
1731         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
1732                 alloc_flags |= AMDGPU_GEM_CREATE_COHERENT;
1733         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT)
1734                 alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT;
1735         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED)
1736                 alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED;
1737
1738         *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1739         if (!*mem) {
1740                 ret = -ENOMEM;
1741                 goto err;
1742         }
1743         INIT_LIST_HEAD(&(*mem)->attachments);
1744         mutex_init(&(*mem)->lock);
1745         (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1746
1747         /* Workaround for AQL queue wraparound bug. Map the same
1748          * memory twice. That means we only actually allocate half
1749          * the memory.
1750          */
1751         if ((*mem)->aql_queue)
1752                 size >>= 1;
1753         aligned_size = PAGE_ALIGN(size);
1754
1755         (*mem)->alloc_flags = flags;
1756
1757         amdgpu_sync_create(&(*mem)->sync);
1758
1759         ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags,
1760                                               xcp_id);
1761         if (ret) {
1762                 pr_debug("Insufficient memory\n");
1763                 goto err_reserve_limit;
1764         }
1765
1766         pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",
1767                  va, (*mem)->aql_queue ? size << 1 : size,
1768                  domain_string(alloc_domain), xcp_id);
1769
1770         ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,
1771                                        bo_type, NULL, &gobj, xcp_id + 1);
1772         if (ret) {
1773                 pr_debug("Failed to create BO on domain %s. ret %d\n",
1774                          domain_string(alloc_domain), ret);
1775                 goto err_bo_create;
1776         }
1777         ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);
1778         if (ret) {
1779                 pr_debug("Failed to allow vma node access. ret %d\n", ret);
1780                 goto err_node_allow;
1781         }
1782         bo = gem_to_amdgpu_bo(gobj);
1783         if (bo_type == ttm_bo_type_sg) {
1784                 bo->tbo.sg = sg;
1785                 bo->tbo.ttm->sg = sg;
1786         }
1787         bo->kfd_bo = *mem;
1788         (*mem)->bo = bo;
1789         if (user_addr)
1790                 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1791
1792         (*mem)->va = va;
1793         (*mem)->domain = domain;
1794         (*mem)->mapped_to_gpu_memory = 0;
1795         (*mem)->process_info = avm->process_info;
1796
1797         add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
1798
1799         if (user_addr) {
1800                 pr_debug("creating userptr BO for user_addr = %llx\n", user_addr);
1801                 ret = init_user_pages(*mem, user_addr, criu_resume);
1802                 if (ret)
1803                         goto allocate_init_user_pages_failed;
1804         } else  if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1805                                 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1806                 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
1807                 if (ret) {
1808                         pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
1809                         goto err_pin_bo;
1810                 }
1811                 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
1812                 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
1813         } else {
1814                 mutex_lock(&avm->process_info->lock);
1815                 if (avm->process_info->eviction_fence &&
1816                     !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))
1817                         ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,
1818                                 &avm->process_info->eviction_fence->base);
1819                 mutex_unlock(&avm->process_info->lock);
1820                 if (ret)
1821                         goto err_validate_bo;
1822         }
1823
1824         if (offset)
1825                 *offset = amdgpu_bo_mmap_offset(bo);
1826
1827         return 0;
1828
1829 allocate_init_user_pages_failed:
1830 err_pin_bo:
1831 err_validate_bo:
1832         remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
1833         drm_vma_node_revoke(&gobj->vma_node, drm_priv);
1834 err_node_allow:
1835         /* Don't unreserve system mem limit twice */
1836         goto err_reserve_limit;
1837 err_bo_create:
1838         amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id);
1839 err_reserve_limit:
1840         mutex_destroy(&(*mem)->lock);
1841         if (gobj)
1842                 drm_gem_object_put(gobj);
1843         else
1844                 kfree(*mem);
1845 err:
1846         if (sg) {
1847                 sg_free_table(sg);
1848                 kfree(sg);
1849         }
1850         return ret;
1851 }
1852
1853 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1854                 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1855                 uint64_t *size)
1856 {
1857         struct amdkfd_process_info *process_info = mem->process_info;
1858         unsigned long bo_size = mem->bo->tbo.base.size;
1859         bool use_release_notifier = (mem->bo->kfd_bo == mem);
1860         struct kfd_mem_attachment *entry, *tmp;
1861         struct bo_vm_reservation_context ctx;
1862         unsigned int mapped_to_gpu_memory;
1863         int ret;
1864         bool is_imported = false;
1865
1866         mutex_lock(&mem->lock);
1867
1868         /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
1869         if (mem->alloc_flags &
1870             (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1871              KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1872                 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
1873         }
1874
1875         mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1876         is_imported = mem->is_imported;
1877         mutex_unlock(&mem->lock);
1878         /* lock is not needed after this, since mem is unused and will
1879          * be freed anyway
1880          */
1881
1882         if (mapped_to_gpu_memory > 0) {
1883                 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1884                                 mem->va, bo_size);
1885                 return -EBUSY;
1886         }
1887
1888         /* Make sure restore workers don't access the BO any more */
1889         mutex_lock(&process_info->lock);
1890         list_del(&mem->validate_list);
1891         mutex_unlock(&process_info->lock);
1892
1893         /* Cleanup user pages and MMU notifiers */
1894         if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
1895                 amdgpu_hmm_unregister(mem->bo);
1896                 mutex_lock(&process_info->notifier_lock);
1897                 amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range);
1898                 mutex_unlock(&process_info->notifier_lock);
1899         }
1900
1901         ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1902         if (unlikely(ret))
1903                 return ret;
1904
1905         amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1906                                         process_info->eviction_fence);
1907         pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1908                 mem->va + bo_size * (1 + mem->aql_queue));
1909
1910         /* Remove from VM internal data structures */
1911         list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {
1912                 kfd_mem_dmaunmap_attachment(mem, entry);
1913                 kfd_mem_detach(entry);
1914         }
1915
1916         ret = unreserve_bo_and_vms(&ctx, false, false);
1917
1918         /* Free the sync object */
1919         amdgpu_sync_free(&mem->sync);
1920
1921         /* If the SG is not NULL, it's one we created for a doorbell or mmio
1922          * remap BO. We need to free it.
1923          */
1924         if (mem->bo->tbo.sg) {
1925                 sg_free_table(mem->bo->tbo.sg);
1926                 kfree(mem->bo->tbo.sg);
1927         }
1928
1929         /* Update the size of the BO being freed if it was allocated from
1930          * VRAM and is not imported. For APP APU VRAM allocations are done
1931          * in GTT domain
1932          */
1933         if (size) {
1934                 if (!is_imported &&
1935                    (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||
1936                    (adev->gmc.is_app_apu &&
1937                     mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))
1938                         *size = bo_size;
1939                 else
1940                         *size = 0;
1941         }
1942
1943         /* Free the BO*/
1944         drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);
1945         if (mem->dmabuf)
1946                 dma_buf_put(mem->dmabuf);
1947         mutex_destroy(&mem->lock);
1948
1949         /* If this releases the last reference, it will end up calling
1950          * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1951          * this needs to be the last call here.
1952          */
1953         drm_gem_object_put(&mem->bo->tbo.base);
1954
1955         /*
1956          * For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),
1957          * explicitly free it here.
1958          */
1959         if (!use_release_notifier)
1960                 kfree(mem);
1961
1962         return ret;
1963 }
1964
1965 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1966                 struct amdgpu_device *adev, struct kgd_mem *mem,
1967                 void *drm_priv)
1968 {
1969         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1970         int ret;
1971         struct amdgpu_bo *bo;
1972         uint32_t domain;
1973         struct kfd_mem_attachment *entry;
1974         struct bo_vm_reservation_context ctx;
1975         unsigned long bo_size;
1976         bool is_invalid_userptr = false;
1977
1978         bo = mem->bo;
1979         if (!bo) {
1980                 pr_err("Invalid BO when mapping memory to GPU\n");
1981                 return -EINVAL;
1982         }
1983
1984         /* Make sure restore is not running concurrently. Since we
1985          * don't map invalid userptr BOs, we rely on the next restore
1986          * worker to do the mapping
1987          */
1988         mutex_lock(&mem->process_info->lock);
1989
1990         /* Lock notifier lock. If we find an invalid userptr BO, we can be
1991          * sure that the MMU notifier is no longer running
1992          * concurrently and the queues are actually stopped
1993          */
1994         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1995                 mutex_lock(&mem->process_info->notifier_lock);
1996                 is_invalid_userptr = !!mem->invalid;
1997                 mutex_unlock(&mem->process_info->notifier_lock);
1998         }
1999
2000         mutex_lock(&mem->lock);
2001
2002         domain = mem->domain;
2003         bo_size = bo->tbo.base.size;
2004
2005         pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
2006                         mem->va,
2007                         mem->va + bo_size * (1 + mem->aql_queue),
2008                         avm, domain_string(domain));
2009
2010         if (!kfd_mem_is_attached(avm, mem)) {
2011                 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue);
2012                 if (ret)
2013                         goto out;
2014         }
2015
2016         ret = reserve_bo_and_vm(mem, avm, &ctx);
2017         if (unlikely(ret))
2018                 goto out;
2019
2020         /* Userptr can be marked as "not invalid", but not actually be
2021          * validated yet (still in the system domain). In that case
2022          * the queues are still stopped and we can leave mapping for
2023          * the next restore worker
2024          */
2025         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
2026             bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
2027                 is_invalid_userptr = true;
2028
2029         ret = vm_validate_pt_pd_bos(avm);
2030         if (unlikely(ret))
2031                 goto out_unreserve;
2032
2033         list_for_each_entry(entry, &mem->attachments, list) {
2034                 if (entry->bo_va->base.vm != avm || entry->is_mapped)
2035                         continue;
2036
2037                 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
2038                          entry->va, entry->va + bo_size, entry);
2039
2040                 ret = map_bo_to_gpuvm(mem, entry, ctx.sync,
2041                                       is_invalid_userptr);
2042                 if (ret) {
2043                         pr_err("Failed to map bo to gpuvm\n");
2044                         goto out_unreserve;
2045                 }
2046
2047                 ret = vm_update_pds(avm, ctx.sync);
2048                 if (ret) {
2049                         pr_err("Failed to update page directories\n");
2050                         goto out_unreserve;
2051                 }
2052
2053                 entry->is_mapped = true;
2054                 mem->mapped_to_gpu_memory++;
2055                 pr_debug("\t INC mapping count %d\n",
2056                          mem->mapped_to_gpu_memory);
2057         }
2058
2059         ret = unreserve_bo_and_vms(&ctx, false, false);
2060
2061         goto out;
2062
2063 out_unreserve:
2064         unreserve_bo_and_vms(&ctx, false, false);
2065 out:
2066         mutex_unlock(&mem->process_info->lock);
2067         mutex_unlock(&mem->lock);
2068         return ret;
2069 }
2070
2071 void amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)
2072 {
2073         struct kfd_mem_attachment *entry;
2074         struct amdgpu_vm *vm;
2075
2076         vm = drm_priv_to_vm(drm_priv);
2077
2078         mutex_lock(&mem->lock);
2079
2080         list_for_each_entry(entry, &mem->attachments, list) {
2081                 if (entry->bo_va->base.vm == vm)
2082                         kfd_mem_dmaunmap_attachment(mem, entry);
2083         }
2084
2085         mutex_unlock(&mem->lock);
2086 }
2087
2088 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
2089                 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2090 {
2091         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2092         unsigned long bo_size = mem->bo->tbo.base.size;
2093         struct kfd_mem_attachment *entry;
2094         struct bo_vm_reservation_context ctx;
2095         int ret;
2096
2097         mutex_lock(&mem->lock);
2098
2099         ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);
2100         if (unlikely(ret))
2101                 goto out;
2102         /* If no VMs were reserved, it means the BO wasn't actually mapped */
2103         if (ctx.n_vms == 0) {
2104                 ret = -EINVAL;
2105                 goto unreserve_out;
2106         }
2107
2108         ret = vm_validate_pt_pd_bos(avm);
2109         if (unlikely(ret))
2110                 goto unreserve_out;
2111
2112         pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
2113                 mem->va,
2114                 mem->va + bo_size * (1 + mem->aql_queue),
2115                 avm);
2116
2117         list_for_each_entry(entry, &mem->attachments, list) {
2118                 if (entry->bo_va->base.vm != avm || !entry->is_mapped)
2119                         continue;
2120
2121                 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
2122                          entry->va, entry->va + bo_size, entry);
2123
2124                 unmap_bo_from_gpuvm(mem, entry, ctx.sync);
2125                 entry->is_mapped = false;
2126
2127                 mem->mapped_to_gpu_memory--;
2128                 pr_debug("\t DEC mapping count %d\n",
2129                          mem->mapped_to_gpu_memory);
2130         }
2131
2132 unreserve_out:
2133         unreserve_bo_and_vms(&ctx, false, false);
2134 out:
2135         mutex_unlock(&mem->lock);
2136         return ret;
2137 }
2138
2139 int amdgpu_amdkfd_gpuvm_sync_memory(
2140                 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2141 {
2142         struct amdgpu_sync sync;
2143         int ret;
2144
2145         amdgpu_sync_create(&sync);
2146
2147         mutex_lock(&mem->lock);
2148         amdgpu_sync_clone(&mem->sync, &sync);
2149         mutex_unlock(&mem->lock);
2150
2151         ret = amdgpu_sync_wait(&sync, intr);
2152         amdgpu_sync_free(&sync);
2153         return ret;
2154 }
2155
2156 /**
2157  * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count
2158  * @adev: Device to which allocated BO belongs
2159  * @bo: Buffer object to be mapped
2160  *
2161  * Before return, bo reference count is incremented. To release the reference and unpin/
2162  * unmap the BO, call amdgpu_amdkfd_free_gtt_mem.
2163  */
2164 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo)
2165 {
2166         int ret;
2167
2168         ret = amdgpu_bo_reserve(bo, true);
2169         if (ret) {
2170                 pr_err("Failed to reserve bo. ret %d\n", ret);
2171                 goto err_reserve_bo_failed;
2172         }
2173
2174         ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2175         if (ret) {
2176                 pr_err("Failed to pin bo. ret %d\n", ret);
2177                 goto err_pin_bo_failed;
2178         }
2179
2180         ret = amdgpu_ttm_alloc_gart(&bo->tbo);
2181         if (ret) {
2182                 pr_err("Failed to bind bo to GART. ret %d\n", ret);
2183                 goto err_map_bo_gart_failed;
2184         }
2185
2186         amdgpu_amdkfd_remove_eviction_fence(
2187                 bo, bo->vm_bo->vm->process_info->eviction_fence);
2188
2189         amdgpu_bo_unreserve(bo);
2190
2191         bo = amdgpu_bo_ref(bo);
2192
2193         return 0;
2194
2195 err_map_bo_gart_failed:
2196         amdgpu_bo_unpin(bo);
2197 err_pin_bo_failed:
2198         amdgpu_bo_unreserve(bo);
2199 err_reserve_bo_failed:
2200
2201         return ret;
2202 }
2203
2204 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
2205  *
2206  * @mem: Buffer object to be mapped for CPU access
2207  * @kptr[out]: pointer in kernel CPU address space
2208  * @size[out]: size of the buffer
2209  *
2210  * Pins the BO and maps it for kernel CPU access. The eviction fence is removed
2211  * from the BO, since pinned BOs cannot be evicted. The bo must remain on the
2212  * validate_list, so the GPU mapping can be restored after a page table was
2213  * evicted.
2214  *
2215  * Return: 0 on success, error code on failure
2216  */
2217 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2218                                              void **kptr, uint64_t *size)
2219 {
2220         int ret;
2221         struct amdgpu_bo *bo = mem->bo;
2222
2223         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
2224                 pr_err("userptr can't be mapped to kernel\n");
2225                 return -EINVAL;
2226         }
2227
2228         mutex_lock(&mem->process_info->lock);
2229
2230         ret = amdgpu_bo_reserve(bo, true);
2231         if (ret) {
2232                 pr_err("Failed to reserve bo. ret %d\n", ret);
2233                 goto bo_reserve_failed;
2234         }
2235
2236         ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2237         if (ret) {
2238                 pr_err("Failed to pin bo. ret %d\n", ret);
2239                 goto pin_failed;
2240         }
2241
2242         ret = amdgpu_bo_kmap(bo, kptr);
2243         if (ret) {
2244                 pr_err("Failed to map bo to kernel. ret %d\n", ret);
2245                 goto kmap_failed;
2246         }
2247
2248         amdgpu_amdkfd_remove_eviction_fence(
2249                 bo, mem->process_info->eviction_fence);
2250
2251         if (size)
2252                 *size = amdgpu_bo_size(bo);
2253
2254         amdgpu_bo_unreserve(bo);
2255
2256         mutex_unlock(&mem->process_info->lock);
2257         return 0;
2258
2259 kmap_failed:
2260         amdgpu_bo_unpin(bo);
2261 pin_failed:
2262         amdgpu_bo_unreserve(bo);
2263 bo_reserve_failed:
2264         mutex_unlock(&mem->process_info->lock);
2265
2266         return ret;
2267 }
2268
2269 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
2270  *
2271  * @mem: Buffer object to be unmapped for CPU access
2272  *
2273  * Removes the kernel CPU mapping and unpins the BO. It does not restore the
2274  * eviction fence, so this function should only be used for cleanup before the
2275  * BO is destroyed.
2276  */
2277 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2278 {
2279         struct amdgpu_bo *bo = mem->bo;
2280
2281         amdgpu_bo_reserve(bo, true);
2282         amdgpu_bo_kunmap(bo);
2283         amdgpu_bo_unpin(bo);
2284         amdgpu_bo_unreserve(bo);
2285 }
2286
2287 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
2288                                           struct kfd_vm_fault_info *mem)
2289 {
2290         if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
2291                 *mem = *adev->gmc.vm_fault_info;
2292                 mb(); /* make sure read happened */
2293                 atomic_set(&adev->gmc.vm_fault_info_updated, 0);
2294         }
2295         return 0;
2296 }
2297
2298 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
2299                                       struct dma_buf *dma_buf,
2300                                       uint64_t va, void *drm_priv,
2301                                       struct kgd_mem **mem, uint64_t *size,
2302                                       uint64_t *mmap_offset)
2303 {
2304         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2305         struct drm_gem_object *obj;
2306         struct amdgpu_bo *bo;
2307         int ret;
2308
2309         obj = amdgpu_gem_prime_import(adev_to_drm(adev), dma_buf);
2310         if (IS_ERR(obj))
2311                 return PTR_ERR(obj);
2312
2313         bo = gem_to_amdgpu_bo(obj);
2314         if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
2315                                     AMDGPU_GEM_DOMAIN_GTT))) {
2316                 /* Only VRAM and GTT BOs are supported */
2317                 ret = -EINVAL;
2318                 goto err_put_obj;
2319         }
2320
2321         *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2322         if (!*mem) {
2323                 ret = -ENOMEM;
2324                 goto err_put_obj;
2325         }
2326
2327         ret = drm_vma_node_allow(&obj->vma_node, drm_priv);
2328         if (ret)
2329                 goto err_free_mem;
2330
2331         if (size)
2332                 *size = amdgpu_bo_size(bo);
2333
2334         if (mmap_offset)
2335                 *mmap_offset = amdgpu_bo_mmap_offset(bo);
2336
2337         INIT_LIST_HEAD(&(*mem)->attachments);
2338         mutex_init(&(*mem)->lock);
2339
2340         (*mem)->alloc_flags =
2341                 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2342                 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)
2343                 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
2344                 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
2345
2346         get_dma_buf(dma_buf);
2347         (*mem)->dmabuf = dma_buf;
2348         (*mem)->bo = bo;
2349         (*mem)->va = va;
2350         (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ?
2351                 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
2352
2353         (*mem)->mapped_to_gpu_memory = 0;
2354         (*mem)->process_info = avm->process_info;
2355         add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
2356         amdgpu_sync_create(&(*mem)->sync);
2357         (*mem)->is_imported = true;
2358
2359         mutex_lock(&avm->process_info->lock);
2360         if (avm->process_info->eviction_fence &&
2361             !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))
2362                 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain,
2363                                 &avm->process_info->eviction_fence->base);
2364         mutex_unlock(&avm->process_info->lock);
2365         if (ret)
2366                 goto err_remove_mem;
2367
2368         return 0;
2369
2370 err_remove_mem:
2371         remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
2372         drm_vma_node_revoke(&obj->vma_node, drm_priv);
2373 err_free_mem:
2374         kfree(*mem);
2375 err_put_obj:
2376         drm_gem_object_put(obj);
2377         return ret;
2378 }
2379
2380 int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
2381                                       struct dma_buf **dma_buf)
2382 {
2383         int ret;
2384
2385         mutex_lock(&mem->lock);
2386         ret = kfd_mem_export_dmabuf(mem);
2387         if (ret)
2388                 goto out;
2389
2390         get_dma_buf(mem->dmabuf);
2391         *dma_buf = mem->dmabuf;
2392 out:
2393         mutex_unlock(&mem->lock);
2394         return ret;
2395 }
2396
2397 /* Evict a userptr BO by stopping the queues if necessary
2398  *
2399  * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
2400  * cannot do any memory allocations, and cannot take any locks that
2401  * are held elsewhere while allocating memory.
2402  *
2403  * It doesn't do anything to the BO itself. The real work happens in
2404  * restore, where we get updated page addresses. This function only
2405  * ensures that GPU access to the BO is stopped.
2406  */
2407 int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,
2408                                 unsigned long cur_seq, struct kgd_mem *mem)
2409 {
2410         struct amdkfd_process_info *process_info = mem->process_info;
2411         int r = 0;
2412
2413         /* Do not process MMU notifications during CRIU restore until
2414          * KFD_CRIU_OP_RESUME IOCTL is received
2415          */
2416         if (READ_ONCE(process_info->block_mmu_notifications))
2417                 return 0;
2418
2419         mutex_lock(&process_info->notifier_lock);
2420         mmu_interval_set_seq(mni, cur_seq);
2421
2422         mem->invalid++;
2423         if (++process_info->evicted_bos == 1) {
2424                 /* First eviction, stop the queues */
2425                 r = kgd2kfd_quiesce_mm(mni->mm,
2426                                        KFD_QUEUE_EVICTION_TRIGGER_USERPTR);
2427                 if (r)
2428                         pr_err("Failed to quiesce KFD\n");
2429                 schedule_delayed_work(&process_info->restore_userptr_work,
2430                         msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2431         }
2432         mutex_unlock(&process_info->notifier_lock);
2433
2434         return r;
2435 }
2436
2437 /* Update invalid userptr BOs
2438  *
2439  * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
2440  * userptr_inval_list and updates user pages for all BOs that have
2441  * been invalidated since their last update.
2442  */
2443 static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
2444                                      struct mm_struct *mm)
2445 {
2446         struct kgd_mem *mem, *tmp_mem;
2447         struct amdgpu_bo *bo;
2448         struct ttm_operation_ctx ctx = { false, false };
2449         uint32_t invalid;
2450         int ret = 0;
2451
2452         mutex_lock(&process_info->notifier_lock);
2453
2454         /* Move all invalidated BOs to the userptr_inval_list */
2455         list_for_each_entry_safe(mem, tmp_mem,
2456                                  &process_info->userptr_valid_list,
2457                                  validate_list)
2458                 if (mem->invalid)
2459                         list_move_tail(&mem->validate_list,
2460                                        &process_info->userptr_inval_list);
2461
2462         /* Go through userptr_inval_list and update any invalid user_pages */
2463         list_for_each_entry(mem, &process_info->userptr_inval_list,
2464                             validate_list) {
2465                 invalid = mem->invalid;
2466                 if (!invalid)
2467                         /* BO hasn't been invalidated since the last
2468                          * revalidation attempt. Keep its page list.
2469                          */
2470                         continue;
2471
2472                 bo = mem->bo;
2473
2474                 amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range);
2475                 mem->range = NULL;
2476
2477                 /* BO reservations and getting user pages (hmm_range_fault)
2478                  * must happen outside the notifier lock
2479                  */
2480                 mutex_unlock(&process_info->notifier_lock);
2481
2482                 /* Move the BO to system (CPU) domain if necessary to unmap
2483                  * and free the SG table
2484                  */
2485                 if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) {
2486                         if (amdgpu_bo_reserve(bo, true))
2487                                 return -EAGAIN;
2488                         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
2489                         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
2490                         amdgpu_bo_unreserve(bo);
2491                         if (ret) {
2492                                 pr_err("%s: Failed to invalidate userptr BO\n",
2493                                        __func__);
2494                                 return -EAGAIN;
2495                         }
2496                 }
2497
2498                 /* Get updated user pages */
2499                 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
2500                                                    &mem->range);
2501                 if (ret) {
2502                         pr_debug("Failed %d to get user pages\n", ret);
2503
2504                         /* Return -EFAULT bad address error as success. It will
2505                          * fail later with a VM fault if the GPU tries to access
2506                          * it. Better than hanging indefinitely with stalled
2507                          * user mode queues.
2508                          *
2509                          * Return other error -EBUSY or -ENOMEM to retry restore
2510                          */
2511                         if (ret != -EFAULT)
2512                                 return ret;
2513
2514                         ret = 0;
2515                 }
2516
2517                 mutex_lock(&process_info->notifier_lock);
2518
2519                 /* Mark the BO as valid unless it was invalidated
2520                  * again concurrently.
2521                  */
2522                 if (mem->invalid != invalid) {
2523                         ret = -EAGAIN;
2524                         goto unlock_out;
2525                 }
2526                  /* set mem valid if mem has hmm range associated */
2527                 if (mem->range)
2528                         mem->invalid = 0;
2529         }
2530
2531 unlock_out:
2532         mutex_unlock(&process_info->notifier_lock);
2533
2534         return ret;
2535 }
2536
2537 /* Validate invalid userptr BOs
2538  *
2539  * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables
2540  * with new page addresses and waits for the page table updates to complete.
2541  */
2542 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
2543 {
2544         struct ttm_operation_ctx ctx = { false, false };
2545         struct amdgpu_sync sync;
2546         struct drm_exec exec;
2547
2548         struct amdgpu_vm *peer_vm;
2549         struct kgd_mem *mem, *tmp_mem;
2550         struct amdgpu_bo *bo;
2551         int ret;
2552
2553         amdgpu_sync_create(&sync);
2554
2555         drm_exec_init(&exec, 0);
2556         /* Reserve all BOs and page tables for validation */
2557         drm_exec_until_all_locked(&exec) {
2558                 /* Reserve all the page directories */
2559                 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2560                                     vm_list_node) {
2561                         ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2);
2562                         drm_exec_retry_on_contention(&exec);
2563                         if (unlikely(ret))
2564                                 goto unreserve_out;
2565                 }
2566
2567                 /* Reserve the userptr_inval_list entries to resv_list */
2568                 list_for_each_entry(mem, &process_info->userptr_inval_list,
2569                                     validate_list) {
2570                         struct drm_gem_object *gobj;
2571
2572                         gobj = &mem->bo->tbo.base;
2573                         ret = drm_exec_prepare_obj(&exec, gobj, 1);
2574                         drm_exec_retry_on_contention(&exec);
2575                         if (unlikely(ret))
2576                                 goto unreserve_out;
2577                 }
2578         }
2579
2580         ret = process_validate_vms(process_info);
2581         if (ret)
2582                 goto unreserve_out;
2583
2584         /* Validate BOs and update GPUVM page tables */
2585         list_for_each_entry_safe(mem, tmp_mem,
2586                                  &process_info->userptr_inval_list,
2587                                  validate_list) {
2588                 struct kfd_mem_attachment *attachment;
2589
2590                 bo = mem->bo;
2591
2592                 /* Validate the BO if we got user pages */
2593                 if (bo->tbo.ttm->pages[0]) {
2594                         amdgpu_bo_placement_from_domain(bo, mem->domain);
2595                         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
2596                         if (ret) {
2597                                 pr_err("%s: failed to validate BO\n", __func__);
2598                                 goto unreserve_out;
2599                         }
2600                 }
2601
2602                 /* Update mapping. If the BO was not validated
2603                  * (because we couldn't get user pages), this will
2604                  * clear the page table entries, which will result in
2605                  * VM faults if the GPU tries to access the invalid
2606                  * memory.
2607                  */
2608                 list_for_each_entry(attachment, &mem->attachments, list) {
2609                         if (!attachment->is_mapped)
2610                                 continue;
2611
2612                         kfd_mem_dmaunmap_attachment(mem, attachment);
2613                         ret = update_gpuvm_pte(mem, attachment, &sync);
2614                         if (ret) {
2615                                 pr_err("%s: update PTE failed\n", __func__);
2616                                 /* make sure this gets validated again */
2617                                 mutex_lock(&process_info->notifier_lock);
2618                                 mem->invalid++;
2619                                 mutex_unlock(&process_info->notifier_lock);
2620                                 goto unreserve_out;
2621                         }
2622                 }
2623         }
2624
2625         /* Update page directories */
2626         ret = process_update_pds(process_info, &sync);
2627
2628 unreserve_out:
2629         drm_exec_fini(&exec);
2630         amdgpu_sync_wait(&sync, false);
2631         amdgpu_sync_free(&sync);
2632
2633         return ret;
2634 }
2635
2636 /* Confirm that all user pages are valid while holding the notifier lock
2637  *
2638  * Moves valid BOs from the userptr_inval_list back to userptr_val_list.
2639  */
2640 static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)
2641 {
2642         struct kgd_mem *mem, *tmp_mem;
2643         int ret = 0;
2644
2645         list_for_each_entry_safe(mem, tmp_mem,
2646                                  &process_info->userptr_inval_list,
2647                                  validate_list) {
2648                 bool valid;
2649
2650                 /* keep mem without hmm range at userptr_inval_list */
2651                 if (!mem->range)
2652                          continue;
2653
2654                 /* Only check mem with hmm range associated */
2655                 valid = amdgpu_ttm_tt_get_user_pages_done(
2656                                         mem->bo->tbo.ttm, mem->range);
2657
2658                 mem->range = NULL;
2659                 if (!valid) {
2660                         WARN(!mem->invalid, "Invalid BO not marked invalid");
2661                         ret = -EAGAIN;
2662                         continue;
2663                 }
2664
2665                 if (mem->invalid) {
2666                         WARN(1, "Valid BO is marked invalid");
2667                         ret = -EAGAIN;
2668                         continue;
2669                 }
2670
2671                 list_move_tail(&mem->validate_list,
2672                                &process_info->userptr_valid_list);
2673         }
2674
2675         return ret;
2676 }
2677
2678 /* Worker callback to restore evicted userptr BOs
2679  *
2680  * Tries to update and validate all userptr BOs. If successful and no
2681  * concurrent evictions happened, the queues are restarted. Otherwise,
2682  * reschedule for another attempt later.
2683  */
2684 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
2685 {
2686         struct delayed_work *dwork = to_delayed_work(work);
2687         struct amdkfd_process_info *process_info =
2688                 container_of(dwork, struct amdkfd_process_info,
2689                              restore_userptr_work);
2690         struct task_struct *usertask;
2691         struct mm_struct *mm;
2692         uint32_t evicted_bos;
2693
2694         mutex_lock(&process_info->notifier_lock);
2695         evicted_bos = process_info->evicted_bos;
2696         mutex_unlock(&process_info->notifier_lock);
2697         if (!evicted_bos)
2698                 return;
2699
2700         /* Reference task and mm in case of concurrent process termination */
2701         usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
2702         if (!usertask)
2703                 return;
2704         mm = get_task_mm(usertask);
2705         if (!mm) {
2706                 put_task_struct(usertask);
2707                 return;
2708         }
2709
2710         mutex_lock(&process_info->lock);
2711
2712         if (update_invalid_user_pages(process_info, mm))
2713                 goto unlock_out;
2714         /* userptr_inval_list can be empty if all evicted userptr BOs
2715          * have been freed. In that case there is nothing to validate
2716          * and we can just restart the queues.
2717          */
2718         if (!list_empty(&process_info->userptr_inval_list)) {
2719                 if (validate_invalid_user_pages(process_info))
2720                         goto unlock_out;
2721         }
2722         /* Final check for concurrent evicton and atomic update. If
2723          * another eviction happens after successful update, it will
2724          * be a first eviction that calls quiesce_mm. The eviction
2725          * reference counting inside KFD will handle this case.
2726          */
2727         mutex_lock(&process_info->notifier_lock);
2728         if (process_info->evicted_bos != evicted_bos)
2729                 goto unlock_notifier_out;
2730
2731         if (confirm_valid_user_pages_locked(process_info)) {
2732                 WARN(1, "User pages unexpectedly invalid");
2733                 goto unlock_notifier_out;
2734         }
2735
2736         process_info->evicted_bos = evicted_bos = 0;
2737
2738         if (kgd2kfd_resume_mm(mm)) {
2739                 pr_err("%s: Failed to resume KFD\n", __func__);
2740                 /* No recovery from this failure. Probably the CP is
2741                  * hanging. No point trying again.
2742                  */
2743         }
2744
2745 unlock_notifier_out:
2746         mutex_unlock(&process_info->notifier_lock);
2747 unlock_out:
2748         mutex_unlock(&process_info->lock);
2749
2750         /* If validation failed, reschedule another attempt */
2751         if (evicted_bos) {
2752                 schedule_delayed_work(&process_info->restore_userptr_work,
2753                         msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2754
2755                 kfd_smi_event_queue_restore_rescheduled(mm);
2756         }
2757         mmput(mm);
2758         put_task_struct(usertask);
2759 }
2760
2761 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
2762  *   KFD process identified by process_info
2763  *
2764  * @process_info: amdkfd_process_info of the KFD process
2765  *
2766  * After memory eviction, restore thread calls this function. The function
2767  * should be called when the Process is still valid. BO restore involves -
2768  *
2769  * 1.  Release old eviction fence and create new one
2770  * 2.  Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
2771  * 3   Use the second PD list and kfd_bo_list to create a list (ctx.list) of
2772  *     BOs that need to be reserved.
2773  * 4.  Reserve all the BOs
2774  * 5.  Validate of PD and PT BOs.
2775  * 6.  Validate all KFD BOs using kfd_bo_list and Map them and add new fence
2776  * 7.  Add fence to all PD and PT BOs.
2777  * 8.  Unreserve all BOs
2778  */
2779 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
2780 {
2781         struct amdkfd_process_info *process_info = info;
2782         struct amdgpu_vm *peer_vm;
2783         struct kgd_mem *mem;
2784         struct amdgpu_amdkfd_fence *new_fence;
2785         struct list_head duplicate_save;
2786         struct amdgpu_sync sync_obj;
2787         unsigned long failed_size = 0;
2788         unsigned long total_size = 0;
2789         struct drm_exec exec;
2790         int ret;
2791
2792         INIT_LIST_HEAD(&duplicate_save);
2793
2794         mutex_lock(&process_info->lock);
2795
2796         drm_exec_init(&exec, 0);
2797         drm_exec_until_all_locked(&exec) {
2798                 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2799                                     vm_list_node) {
2800                         ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2);
2801                         drm_exec_retry_on_contention(&exec);
2802                         if (unlikely(ret))
2803                                 goto ttm_reserve_fail;
2804                 }
2805
2806                 /* Reserve all BOs and page tables/directory. Add all BOs from
2807                  * kfd_bo_list to ctx.list
2808                  */
2809                 list_for_each_entry(mem, &process_info->kfd_bo_list,
2810                                     validate_list) {
2811                         struct drm_gem_object *gobj;
2812
2813                         gobj = &mem->bo->tbo.base;
2814                         ret = drm_exec_prepare_obj(&exec, gobj, 1);
2815                         drm_exec_retry_on_contention(&exec);
2816                         if (unlikely(ret))
2817                                 goto ttm_reserve_fail;
2818                 }
2819         }
2820
2821         amdgpu_sync_create(&sync_obj);
2822
2823         /* Validate PDs and PTs */
2824         ret = process_validate_vms(process_info);
2825         if (ret)
2826                 goto validate_map_fail;
2827
2828         ret = process_sync_pds_resv(process_info, &sync_obj);
2829         if (ret) {
2830                 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2831                 goto validate_map_fail;
2832         }
2833
2834         /* Validate BOs and map them to GPUVM (update VM page tables). */
2835         list_for_each_entry(mem, &process_info->kfd_bo_list,
2836                             validate_list) {
2837
2838                 struct amdgpu_bo *bo = mem->bo;
2839                 uint32_t domain = mem->domain;
2840                 struct kfd_mem_attachment *attachment;
2841                 struct dma_resv_iter cursor;
2842                 struct dma_fence *fence;
2843
2844                 total_size += amdgpu_bo_size(bo);
2845
2846                 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
2847                 if (ret) {
2848                         pr_debug("Memory eviction: Validate BOs failed\n");
2849                         failed_size += amdgpu_bo_size(bo);
2850                         ret = amdgpu_amdkfd_bo_validate(bo,
2851                                                 AMDGPU_GEM_DOMAIN_GTT, false);
2852                         if (ret) {
2853                                 pr_debug("Memory eviction: Try again\n");
2854                                 goto validate_map_fail;
2855                         }
2856                 }
2857                 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
2858                                         DMA_RESV_USAGE_KERNEL, fence) {
2859                         ret = amdgpu_sync_fence(&sync_obj, fence);
2860                         if (ret) {
2861                                 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2862                                 goto validate_map_fail;
2863                         }
2864                 }
2865                 list_for_each_entry(attachment, &mem->attachments, list) {
2866                         if (!attachment->is_mapped)
2867                                 continue;
2868
2869                         if (attachment->bo_va->base.bo->tbo.pin_count)
2870                                 continue;
2871
2872                         kfd_mem_dmaunmap_attachment(mem, attachment);
2873                         ret = update_gpuvm_pte(mem, attachment, &sync_obj);
2874                         if (ret) {
2875                                 pr_debug("Memory eviction: update PTE failed. Try again\n");
2876                                 goto validate_map_fail;
2877                         }
2878                 }
2879         }
2880
2881         if (failed_size)
2882                 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
2883
2884         /* Update page directories */
2885         ret = process_update_pds(process_info, &sync_obj);
2886         if (ret) {
2887                 pr_debug("Memory eviction: update PDs failed. Try again\n");
2888                 goto validate_map_fail;
2889         }
2890
2891         /* Wait for validate and PT updates to finish */
2892         amdgpu_sync_wait(&sync_obj, false);
2893
2894         /* Release old eviction fence and create new one, because fence only
2895          * goes from unsignaled to signaled, fence cannot be reused.
2896          * Use context and mm from the old fence.
2897          */
2898         new_fence = amdgpu_amdkfd_fence_create(
2899                                 process_info->eviction_fence->base.context,
2900                                 process_info->eviction_fence->mm,
2901                                 NULL);
2902         if (!new_fence) {
2903                 pr_err("Failed to create eviction fence\n");
2904                 ret = -ENOMEM;
2905                 goto validate_map_fail;
2906         }
2907         dma_fence_put(&process_info->eviction_fence->base);
2908         process_info->eviction_fence = new_fence;
2909         *ef = dma_fence_get(&new_fence->base);
2910
2911         /* Attach new eviction fence to all BOs except pinned ones */
2912         list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {
2913                 if (mem->bo->tbo.pin_count)
2914                         continue;
2915
2916                 dma_resv_add_fence(mem->bo->tbo.base.resv,
2917                                    &process_info->eviction_fence->base,
2918                                    DMA_RESV_USAGE_BOOKKEEP);
2919         }
2920         /* Attach eviction fence to PD / PT BOs */
2921         list_for_each_entry(peer_vm, &process_info->vm_list_head,
2922                             vm_list_node) {
2923                 struct amdgpu_bo *bo = peer_vm->root.bo;
2924
2925                 dma_resv_add_fence(bo->tbo.base.resv,
2926                                    &process_info->eviction_fence->base,
2927                                    DMA_RESV_USAGE_BOOKKEEP);
2928         }
2929
2930 validate_map_fail:
2931         amdgpu_sync_free(&sync_obj);
2932 ttm_reserve_fail:
2933         drm_exec_fini(&exec);
2934         mutex_unlock(&process_info->lock);
2935         return ret;
2936 }
2937
2938 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
2939 {
2940         struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2941         struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
2942         int ret;
2943
2944         if (!info || !gws)
2945                 return -EINVAL;
2946
2947         *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2948         if (!*mem)
2949                 return -ENOMEM;
2950
2951         mutex_init(&(*mem)->lock);
2952         INIT_LIST_HEAD(&(*mem)->attachments);
2953         (*mem)->bo = amdgpu_bo_ref(gws_bo);
2954         (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
2955         (*mem)->process_info = process_info;
2956         add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
2957         amdgpu_sync_create(&(*mem)->sync);
2958
2959
2960         /* Validate gws bo the first time it is added to process */
2961         mutex_lock(&(*mem)->process_info->lock);
2962         ret = amdgpu_bo_reserve(gws_bo, false);
2963         if (unlikely(ret)) {
2964                 pr_err("Reserve gws bo failed %d\n", ret);
2965                 goto bo_reservation_failure;
2966         }
2967
2968         ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);
2969         if (ret) {
2970                 pr_err("GWS BO validate failed %d\n", ret);
2971                 goto bo_validation_failure;
2972         }
2973         /* GWS resource is shared b/t amdgpu and amdkfd
2974          * Add process eviction fence to bo so they can
2975          * evict each other.
2976          */
2977         ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1);
2978         if (ret)
2979                 goto reserve_shared_fail;
2980         dma_resv_add_fence(gws_bo->tbo.base.resv,
2981                            &process_info->eviction_fence->base,
2982                            DMA_RESV_USAGE_BOOKKEEP);
2983         amdgpu_bo_unreserve(gws_bo);
2984         mutex_unlock(&(*mem)->process_info->lock);
2985
2986         return ret;
2987
2988 reserve_shared_fail:
2989 bo_validation_failure:
2990         amdgpu_bo_unreserve(gws_bo);
2991 bo_reservation_failure:
2992         mutex_unlock(&(*mem)->process_info->lock);
2993         amdgpu_sync_free(&(*mem)->sync);
2994         remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
2995         amdgpu_bo_unref(&gws_bo);
2996         mutex_destroy(&(*mem)->lock);
2997         kfree(*mem);
2998         *mem = NULL;
2999         return ret;
3000 }
3001
3002 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
3003 {
3004         int ret;
3005         struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3006         struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
3007         struct amdgpu_bo *gws_bo = kgd_mem->bo;
3008
3009         /* Remove BO from process's validate list so restore worker won't touch
3010          * it anymore
3011          */
3012         remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
3013
3014         ret = amdgpu_bo_reserve(gws_bo, false);
3015         if (unlikely(ret)) {
3016                 pr_err("Reserve gws bo failed %d\n", ret);
3017                 //TODO add BO back to validate_list?
3018                 return ret;
3019         }
3020         amdgpu_amdkfd_remove_eviction_fence(gws_bo,
3021                         process_info->eviction_fence);
3022         amdgpu_bo_unreserve(gws_bo);
3023         amdgpu_sync_free(&kgd_mem->sync);
3024         amdgpu_bo_unref(&gws_bo);
3025         mutex_destroy(&kgd_mem->lock);
3026         kfree(mem);
3027         return 0;
3028 }
3029
3030 /* Returns GPU-specific tiling mode information */
3031 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
3032                                 struct tile_config *config)
3033 {
3034         config->gb_addr_config = adev->gfx.config.gb_addr_config;
3035         config->tile_config_ptr = adev->gfx.config.tile_mode_array;
3036         config->num_tile_configs =
3037                         ARRAY_SIZE(adev->gfx.config.tile_mode_array);
3038         config->macro_tile_config_ptr =
3039                         adev->gfx.config.macrotile_mode_array;
3040         config->num_macro_tile_configs =
3041                         ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
3042
3043         /* Those values are not set from GFX9 onwards */
3044         config->num_banks = adev->gfx.config.num_banks;
3045         config->num_ranks = adev->gfx.config.num_ranks;
3046
3047         return 0;
3048 }
3049
3050 bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem)
3051 {
3052         struct kfd_mem_attachment *entry;
3053
3054         list_for_each_entry(entry, &mem->attachments, list) {
3055                 if (entry->is_mapped && entry->adev == adev)
3056                         return true;
3057         }
3058         return false;
3059 }
3060
3061 #if defined(CONFIG_DEBUG_FS)
3062
3063 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)
3064 {
3065
3066         spin_lock(&kfd_mem_limit.mem_limit_lock);
3067         seq_printf(m, "System mem used %lldM out of %lluM\n",
3068                   (kfd_mem_limit.system_mem_used >> 20),
3069                   (kfd_mem_limit.max_system_mem_limit >> 20));
3070         seq_printf(m, "TTM mem used %lldM out of %lluM\n",
3071                   (kfd_mem_limit.ttm_mem_used >> 20),
3072                   (kfd_mem_limit.max_ttm_mem_limit >> 20));
3073         spin_unlock(&kfd_mem_limit.mem_limit_lock);
3074
3075         return 0;
3076 }
3077
3078 #endif