drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <glisse@freedesktop.org>
  29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30  *    Dave Airlie
  31  */
  32
  33 #include <linux/dma-mapping.h>
  34 #include <linux/iommu.h>
  35 #include <linux/hmm.h>
  36 #include <linux/pagemap.h>
  37 #include <linux/sched/task.h>
  38 #include <linux/sched/mm.h>
  39 #include <linux/seq_file.h>
  40 #include <linux/slab.h>
  41 #include <linux/swap.h>
  42 #include <linux/swiotlb.h>
  43 #include <linux/dma-buf.h>
  44 #include <linux/sizes.h>
  45
  46 #include <drm/ttm/ttm_bo_api.h>
  47 #include <drm/ttm/ttm_bo_driver.h>
  48 #include <drm/ttm/ttm_placement.h>
  49
  50 #include <drm/amdgpu_drm.h>
  51
  52 #include "amdgpu.h"
  53 #include "amdgpu_object.h"
  54 #include "amdgpu_trace.h"
  55 #include "amdgpu_amdkfd.h"
  56 #include "amdgpu_sdma.h"
  57 #include "amdgpu_ras.h"
  58 #include "amdgpu_atomfirmware.h"
  59 #include "amdgpu_res_cursor.h"
  60 #include "bif/bif_4_1_d.h"
  61
  62 #define AMDGPU_TTM_VRAM_MAX_DW_READ     (size_t)128
  63
  64 static int amdgpu_ttm_backend_bind(struct ttm_bo_device *bdev,
  65                                    struct ttm_tt *ttm,
  66                                    struct ttm_resource *bo_mem);
  67 static void amdgpu_ttm_backend_unbind(struct ttm_bo_device *bdev,
  68                                       struct ttm_tt *ttm);
  69
  70 static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev,
  71                                     unsigned int type,
  72                                     uint64_t size_in_page)
  73 {
  74         return ttm_range_man_init(&adev->mman.bdev, type,
  75                                   false, size_in_page);
  76 }
  77
  78 /**
  79  * amdgpu_evict_flags - Compute placement flags
  80  *
  81  * @bo: The buffer object to evict
  82  * @placement: Possible destination(s) for evicted BO
  83  *
  84  * Fill in placement data when ttm_bo_evict() is called
  85  */
  86 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
  87                                 struct ttm_placement *placement)
  88 {
  89         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
  90         struct amdgpu_bo *abo;
  91         static const struct ttm_place placements = {
  92                 .fpfn = 0,
  93                 .lpfn = 0,
  94                 .mem_type = TTM_PL_SYSTEM,
  95                 .flags = 0
  96         };
  97
  98         /* Don't handle scatter gather BOs */
  99         if (bo->type == ttm_bo_type_sg) {
 100                 placement->num_placement = 0;
 101                 placement->num_busy_placement = 0;
 102                 return;
 103         }
 104
 105         /* Object isn't an AMDGPU object so ignore */
 106         if (!amdgpu_bo_is_amdgpu_bo(bo)) {
 107                 placement->placement = &placements;
 108                 placement->busy_placement = &placements;
 109                 placement->num_placement = 1;
 110                 placement->num_busy_placement = 1;
 111                 return;
 112         }
 113
 114         abo = ttm_to_amdgpu_bo(bo);
 115         switch (bo->mem.mem_type) {
 116         case AMDGPU_PL_GDS:
 117         case AMDGPU_PL_GWS:
 118         case AMDGPU_PL_OA:
 119                 placement->num_placement = 0;
 120                 placement->num_busy_placement = 0;
 121                 return;
 122
 123         case TTM_PL_VRAM:
 124                 if (!adev->mman.buffer_funcs_enabled) {
 125                         /* Move to system memory */
 126                         amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 127                 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 128                            !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
 129                            amdgpu_bo_in_cpu_visible_vram(abo)) {
 130
 131                         /* Try evicting to the CPU inaccessible part of VRAM
 132                          * first, but only set GTT as busy placement, so this
 133                          * BO will be evicted to GTT rather than causing other
 134                          * BOs to be evicted from VRAM
 135                          */
 136                         amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
 137                                                          AMDGPU_GEM_DOMAIN_GTT);
 138                         abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
 139                         abo->placements[0].lpfn = 0;
 140                         abo->placement.busy_placement = &abo->placements[1];
 141                         abo->placement.num_busy_placement = 1;
 142                 } else {
 143                         /* Move to GTT memory */
 144                         amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
 145                 }
 146                 break;
 147         case TTM_PL_TT:
 148         default:
 149                 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 150                 break;
 151         }
 152         *placement = abo->placement;
 153 }
 154
 155 /**
 156  * amdgpu_verify_access - Verify access for a mmap call
 157  *
 158  * @bo: The buffer object to map
 159  * @filp: The file pointer from the process performing the mmap
 160  *
 161  * This is called by ttm_bo_mmap() to verify whether a process
 162  * has the right to mmap a BO to their process space.
 163  */
 164 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 165 {
 166         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
 167
 168         /*
 169          * Don't verify access for KFD BOs. They don't have a GEM
 170          * object associated with them.
 171          */
 172         if (abo->kfd_bo)
 173                 return 0;
 174
 175         if (amdgpu_ttm_tt_get_usermm(bo->ttm))
 176                 return -EPERM;
 177         return drm_vma_node_verify_access(&abo->tbo.base.vma_node,
 178                                           filp->private_data);
 179 }
 180
 181 /**
 182  * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to
 183  * @offset. It also modifies the offset to be within the drm_mm_node returned
 184  *
 185  * @mem: The region where the bo resides.
 186  * @offset: The offset that drm_mm_node is used for finding.
 187  *
 188  */
 189 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_resource *mem,
 190                                                uint64_t *offset)
 191 {
 192         struct drm_mm_node *mm_node = mem->mm_node;
 193
 194         while (*offset >= (mm_node->size << PAGE_SHIFT)) {
 195                 *offset -= (mm_node->size << PAGE_SHIFT);
 196                 ++mm_node;
 197         }
 198         return mm_node;
 199 }
 200
 201 /**
 202  * amdgpu_ttm_map_buffer - Map memory into the GART windows
 203  * @bo: buffer object to map
 204  * @mem: memory object to map
 205  * @mm_cur: range to map
 206  * @num_pages: number of pages to map
 207  * @window: which GART window to use
 208  * @ring: DMA ring to use for the copy
 209  * @tmz: if we should setup a TMZ enabled mapping
 210  * @addr: resulting address inside the MC address space
 211  *
 212  * Setup one of the GART windows to access a specific piece of memory or return
 213  * the physical address for local memory.
 214  */
 215 static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
 216                                  struct ttm_resource *mem,
 217                                  struct amdgpu_res_cursor *mm_cur,
 218                                  unsigned num_pages, unsigned window,
 219                                  struct amdgpu_ring *ring, bool tmz,
 220                                  uint64_t *addr)
 221 {
 222         struct amdgpu_device *adev = ring->adev;
 223         struct amdgpu_job *job;
 224         unsigned num_dw, num_bytes;
 225         struct dma_fence *fence;
 226         uint64_t src_addr, dst_addr;
 227         void *cpu_addr;
 228         uint64_t flags;
 229         unsigned int i;
 230         int r;
 231
 232         BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
 233                AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
 234
 235         /* Map only what can't be accessed directly */
 236         if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) {
 237                 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) +
 238                         mm_cur->start;
 239                 return 0;
 240         }
 241
 242         *addr = adev->gmc.gart_start;
 243         *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
 244                 AMDGPU_GPU_PAGE_SIZE;
 245         *addr += mm_cur->start & ~PAGE_MASK;
 246
 247         num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
 248         num_bytes = num_pages * 8;
 249
 250         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes,
 251                                      AMDGPU_IB_POOL_DELAYED, &job);
 252         if (r)
 253                 return r;
 254
 255         src_addr = num_dw * 4;
 256         src_addr += job->ibs[0].gpu_addr;
 257
 258         dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
 259         dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
 260         amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
 261                                 dst_addr, num_bytes, false);
 262
 263         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 264         WARN_ON(job->ibs[0].length_dw > num_dw);
 265
 266         flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem);
 267         if (tmz)
 268                 flags |= AMDGPU_PTE_TMZ;
 269
 270         cpu_addr = &job->ibs[0].ptr[num_dw];
 271
 272         if (mem->mem_type == TTM_PL_TT) {
 273                 dma_addr_t *dma_addr;
 274
 275                 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT];
 276                 r = amdgpu_gart_map(adev, 0, num_pages, dma_addr, flags,
 277                                     cpu_addr);
 278                 if (r)
 279                         goto error_free;
 280         } else {
 281                 dma_addr_t dma_address;
 282
 283                 dma_address = mm_cur->start;
 284                 dma_address += adev->vm_manager.vram_base_offset;
 285
 286                 for (i = 0; i < num_pages; ++i) {
 287                         r = amdgpu_gart_map(adev, i << PAGE_SHIFT, 1,
 288                                             &dma_address, flags, cpu_addr);
 289                         if (r)
 290                                 goto error_free;
 291
 292                         dma_address += PAGE_SIZE;
 293                 }
 294         }
 295
 296         r = amdgpu_job_submit(job, &adev->mman.entity,
 297                               AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
 298         if (r)
 299                 goto error_free;
 300
 301         dma_fence_put(fence);
 302
 303         return r;
 304
 305 error_free:
 306         amdgpu_job_free(job);
 307         return r;
 308 }
 309
 310 /**
 311  * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
 312  * @adev: amdgpu device
 313  * @src: buffer/address where to read from
 314  * @dst: buffer/address where to write to
 315  * @size: number of bytes to copy
 316  * @tmz: if a secure copy should be used
 317  * @resv: resv object to sync to
 318  * @f: Returns the last fence if multiple jobs are submitted.
 319  *
 320  * The function copies @size bytes from {src->mem + src->offset} to
 321  * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
 322  * move and different for a BO to BO copy.
 323  *
 324  */
 325 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 326                                const struct amdgpu_copy_mem *src,
 327                                const struct amdgpu_copy_mem *dst,
 328                                uint64_t size, bool tmz,
 329                                struct dma_resv *resv,
 330                                struct dma_fence **f)
 331 {
 332         const uint32_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
 333                                         AMDGPU_GPU_PAGE_SIZE);
 334
 335         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 336         struct amdgpu_res_cursor src_mm, dst_mm;
 337         struct dma_fence *fence = NULL;
 338         int r = 0;
 339
 340         if (!adev->mman.buffer_funcs_enabled) {
 341                 DRM_ERROR("Trying to move memory with ring turned off.\n");
 342                 return -EINVAL;
 343         }
 344
 345         amdgpu_res_first(src->mem, src->offset, size, &src_mm);
 346         amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm);
 347
 348         mutex_lock(&adev->mman.gtt_window_lock);
 349         while (src_mm.remaining) {
 350                 uint32_t src_page_offset = src_mm.start & ~PAGE_MASK;
 351                 uint32_t dst_page_offset = dst_mm.start & ~PAGE_MASK;
 352                 struct dma_fence *next;
 353                 uint32_t cur_size;
 354                 uint64_t from, to;
 355
 356                 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
 357                  * begins at an offset, then adjust the size accordingly
 358                  */
 359                 cur_size = max(src_page_offset, dst_page_offset);
 360                 cur_size = min(min3(src_mm.size, dst_mm.size, size),
 361                                (uint64_t)(GTT_MAX_BYTES - cur_size));
 362
 363                 /* Map src to window 0 and dst to window 1. */
 364                 r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm,
 365                                           PFN_UP(cur_size + src_page_offset),
 366                                           0, ring, tmz, &from);
 367                 if (r)
 368                         goto error;
 369
 370                 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm,
 371                                           PFN_UP(cur_size + dst_page_offset),
 372                                           1, ring, tmz, &to);
 373                 if (r)
 374                         goto error;
 375
 376                 r = amdgpu_copy_buffer(ring, from, to, cur_size,
 377                                        resv, &next, false, true, tmz);
 378                 if (r)
 379                         goto error;
 380
 381                 dma_fence_put(fence);
 382                 fence = next;
 383
 384                 amdgpu_res_next(&src_mm, cur_size);
 385                 amdgpu_res_next(&dst_mm, cur_size);
 386         }
 387 error:
 388         mutex_unlock(&adev->mman.gtt_window_lock);
 389         if (f)
 390                 *f = dma_fence_get(fence);
 391         dma_fence_put(fence);
 392         return r;
 393 }
 394
 395 /*
 396  * amdgpu_move_blit - Copy an entire buffer to another buffer
 397  *
 398  * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
 399  * help move buffers to and from VRAM.
 400  */
 401 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 402                             bool evict,
 403                             struct ttm_resource *new_mem,
 404                             struct ttm_resource *old_mem)
 405 {
 406         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 407         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
 408         struct amdgpu_copy_mem src, dst;
 409         struct dma_fence *fence = NULL;
 410         int r;
 411
 412         src.bo = bo;
 413         dst.bo = bo;
 414         src.mem = old_mem;
 415         dst.mem = new_mem;
 416         src.offset = 0;
 417         dst.offset = 0;
 418
 419         r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
 420                                        new_mem->num_pages << PAGE_SHIFT,
 421                                        amdgpu_bo_encrypted(abo),
 422                                        bo->base.resv, &fence);
 423         if (r)
 424                 goto error;
 425
 426         /* clear the space being freed */
 427         if (old_mem->mem_type == TTM_PL_VRAM &&
 428             (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
 429                 struct dma_fence *wipe_fence = NULL;
 430
 431                 r = amdgpu_fill_buffer(ttm_to_amdgpu_bo(bo), AMDGPU_POISON,
 432                                        NULL, &wipe_fence);
 433                 if (r) {
 434                         goto error;
 435                 } else if (wipe_fence) {
 436                         dma_fence_put(fence);
 437                         fence = wipe_fence;
 438                 }
 439         }
 440
 441         /* Always block for VM page tables before committing the new location */
 442         if (bo->type == ttm_bo_type_kernel)
 443                 r = ttm_bo_move_accel_cleanup(bo, fence, true, false, new_mem);
 444         else
 445                 r = ttm_bo_move_accel_cleanup(bo, fence, evict, true, new_mem);
 446         dma_fence_put(fence);
 447         return r;
 448
 449 error:
 450         if (fence)
 451                 dma_fence_wait(fence, false);
 452         dma_fence_put(fence);
 453         return r;
 454 }
 455
 456 /*
 457  * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
 458  *
 459  * Called by amdgpu_bo_move()
 460  */
 461 static bool amdgpu_mem_visible(struct amdgpu_device *adev,
 462                                struct ttm_resource *mem)
 463 {
 464         struct drm_mm_node *nodes = mem->mm_node;
 465
 466         if (mem->mem_type == TTM_PL_SYSTEM ||
 467             mem->mem_type == TTM_PL_TT)
 468                 return true;
 469         if (mem->mem_type != TTM_PL_VRAM)
 470                 return false;
 471
 472         /* ttm_resource_ioremap only supports contiguous memory */
 473         if (nodes->size != mem->num_pages)
 474                 return false;
 475
 476         return ((nodes->start + nodes->size) << PAGE_SHIFT)
 477                 <= adev->gmc.visible_vram_size;
 478 }
 479
 480 /*
 481  * amdgpu_bo_move - Move a buffer object to a new memory location
 482  *
 483  * Called by ttm_bo_handle_move_mem()
 484  */
 485 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 486                           struct ttm_operation_ctx *ctx,
 487                           struct ttm_resource *new_mem,
 488                           struct ttm_place *hop)
 489 {
 490         struct amdgpu_device *adev;
 491         struct amdgpu_bo *abo;
 492         struct ttm_resource *old_mem = &bo->mem;
 493         int r;
 494
 495         if (new_mem->mem_type == TTM_PL_TT) {
 496                 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem);
 497                 if (r)
 498                         return r;
 499         }
 500
 501         /* Can't move a pinned BO */
 502         abo = ttm_to_amdgpu_bo(bo);
 503         if (WARN_ON_ONCE(abo->tbo.pin_count > 0))
 504                 return -EINVAL;
 505
 506         adev = amdgpu_ttm_adev(bo->bdev);
 507
 508         if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
 509                 ttm_bo_move_null(bo, new_mem);
 510                 goto out;
 511         }
 512         if (old_mem->mem_type == TTM_PL_SYSTEM &&
 513             new_mem->mem_type == TTM_PL_TT) {
 514                 ttm_bo_move_null(bo, new_mem);
 515                 goto out;
 516         }
 517         if (old_mem->mem_type == TTM_PL_TT &&
 518             new_mem->mem_type == TTM_PL_SYSTEM) {
 519                 r = ttm_bo_wait_ctx(bo, ctx);
 520                 if (r)
 521                         return r;
 522
 523                 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm);
 524                 ttm_resource_free(bo, &bo->mem);
 525                 ttm_bo_assign_mem(bo, new_mem);
 526                 goto out;
 527         }
 528
 529         if (old_mem->mem_type == AMDGPU_PL_GDS ||
 530             old_mem->mem_type == AMDGPU_PL_GWS ||
 531             old_mem->mem_type == AMDGPU_PL_OA ||
 532             new_mem->mem_type == AMDGPU_PL_GDS ||
 533             new_mem->mem_type == AMDGPU_PL_GWS ||
 534             new_mem->mem_type == AMDGPU_PL_OA) {
 535                 /* Nothing to save here */
 536                 ttm_bo_move_null(bo, new_mem);
 537                 goto out;
 538         }
 539
 540         if (adev->mman.buffer_funcs_enabled) {
 541                 if (((old_mem->mem_type == TTM_PL_SYSTEM &&
 542                       new_mem->mem_type == TTM_PL_VRAM) ||
 543                      (old_mem->mem_type == TTM_PL_VRAM &&
 544                       new_mem->mem_type == TTM_PL_SYSTEM))) {
 545                         hop->fpfn = 0;
 546                         hop->lpfn = 0;
 547                         hop->mem_type = TTM_PL_TT;
 548                         hop->flags = 0;
 549                         return -EMULTIHOP;
 550                 }
 551
 552                 r = amdgpu_move_blit(bo, evict, new_mem, old_mem);
 553         } else {
 554                 r = -ENODEV;
 555         }
 556
 557         if (r) {
 558                 /* Check that all memory is CPU accessible */
 559                 if (!amdgpu_mem_visible(adev, old_mem) ||
 560                     !amdgpu_mem_visible(adev, new_mem)) {
 561                         pr_err("Move buffer fallback to memcpy unavailable\n");
 562                         return r;
 563                 }
 564
 565                 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
 566                 if (r)
 567                         return r;
 568         }
 569
 570         if (bo->type == ttm_bo_type_device &&
 571             new_mem->mem_type == TTM_PL_VRAM &&
 572             old_mem->mem_type != TTM_PL_VRAM) {
 573                 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
 574                  * accesses the BO after it's moved.
 575                  */
 576                 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 577         }
 578
 579 out:
 580         /* update statistics */
 581         atomic64_add(bo->base.size, &adev->num_bytes_moved);
 582         amdgpu_bo_move_notify(bo, evict, new_mem);
 583         return 0;
 584 }
 585
 586 /*
 587  * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
 588  *
 589  * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
 590  */
 591 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem)
 592 {
 593         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 594         struct drm_mm_node *mm_node = mem->mm_node;
 595         size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
 596
 597         switch (mem->mem_type) {
 598         case TTM_PL_SYSTEM:
 599                 /* system memory */
 600                 return 0;
 601         case TTM_PL_TT:
 602                 break;
 603         case TTM_PL_VRAM:
 604                 mem->bus.offset = mem->start << PAGE_SHIFT;
 605                 /* check if it's visible */
 606                 if ((mem->bus.offset + bus_size) > adev->gmc.visible_vram_size)
 607                         return -EINVAL;
 608                 /* Only physically contiguous buffers apply. In a contiguous
 609                  * buffer, size of the first mm_node would match the number of
 610                  * pages in ttm_resource.
 611                  */
 612                 if (adev->mman.aper_base_kaddr &&
 613                     (mm_node->size == mem->num_pages))
 614                         mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
 615                                         mem->bus.offset;
 616
 617                 mem->bus.offset += adev->gmc.aper_base;
 618                 mem->bus.is_iomem = true;
 619                 if (adev->gmc.xgmi.connected_to_cpu)
 620                         mem->bus.caching = ttm_cached;
 621                 else
 622                         mem->bus.caching = ttm_write_combined;
 623                 break;
 624         default:
 625                 return -EINVAL;
 626         }
 627         return 0;
 628 }
 629
 630 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
 631                                            unsigned long page_offset)
 632 {
 633         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 634         uint64_t offset = (page_offset << PAGE_SHIFT);
 635         struct drm_mm_node *mm;
 636
 637         mm = amdgpu_find_mm_node(&bo->mem, &offset);
 638         offset += adev->gmc.aper_base;
 639         return mm->start + (offset >> PAGE_SHIFT);
 640 }
 641
 642 /**
 643  * amdgpu_ttm_domain_start - Returns GPU start address
 644  * @adev: amdgpu device object
 645  * @type: type of the memory
 646  *
 647  * Returns:
 648  * GPU start address of a memory domain
 649  */
 650
 651 uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type)
 652 {
 653         switch (type) {
 654         case TTM_PL_TT:
 655                 return adev->gmc.gart_start;
 656         case TTM_PL_VRAM:
 657                 return adev->gmc.vram_start;
 658         }
 659
 660         return 0;
 661 }
 662
 663 /*
 664  * TTM backend functions.
 665  */
 666 struct amdgpu_ttm_tt {
 667         struct ttm_tt   ttm;
 668         struct drm_gem_object   *gobj;
 669         u64                     offset;
 670         uint64_t                userptr;
 671         struct task_struct      *usertask;
 672         uint32_t                userflags;
 673         bool                    bound;
 674 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
 675         struct hmm_range        *range;
 676 #endif
 677 };
 678
 679 #ifdef CONFIG_DRM_AMDGPU_USERPTR
 680 /*
 681  * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
 682  * memory and start HMM tracking CPU page table update
 683  *
 684  * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
 685  * once afterwards to stop HMM tracking
 686  */
 687 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
 688 {
 689         struct ttm_tt *ttm = bo->tbo.ttm;
 690         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 691         unsigned long start = gtt->userptr;
 692         struct vm_area_struct *vma;
 693         struct hmm_range *range;
 694         unsigned long timeout;
 695         struct mm_struct *mm;
 696         unsigned long i;
 697         int r = 0;
 698
 699         mm = bo->notifier.mm;
 700         if (unlikely(!mm)) {
 701                 DRM_DEBUG_DRIVER("BO is not registered?\n");
 702                 return -EFAULT;
 703         }
 704
 705         /* Another get_user_pages is running at the same time?? */
 706         if (WARN_ON(gtt->range))
 707                 return -EFAULT;
 708
 709         if (!mmget_not_zero(mm)) /* Happens during process shutdown */
 710                 return -ESRCH;
 711
 712         range = kzalloc(sizeof(*range), GFP_KERNEL);
 713         if (unlikely(!range)) {
 714                 r = -ENOMEM;
 715                 goto out;
 716         }
 717         range->notifier = &bo->notifier;
 718         range->start = bo->notifier.interval_tree.start;
 719         range->end = bo->notifier.interval_tree.last + 1;
 720         range->default_flags = HMM_PFN_REQ_FAULT;
 721         if (!amdgpu_ttm_tt_is_readonly(ttm))
 722                 range->default_flags |= HMM_PFN_REQ_WRITE;
 723
 724         range->hmm_pfns = kvmalloc_array(ttm->num_pages,
 725                                          sizeof(*range->hmm_pfns), GFP_KERNEL);
 726         if (unlikely(!range->hmm_pfns)) {
 727                 r = -ENOMEM;
 728                 goto out_free_ranges;
 729         }
 730
 731         mmap_read_lock(mm);
 732         vma = find_vma(mm, start);
 733         if (unlikely(!vma || start < vma->vm_start)) {
 734                 r = -EFAULT;
 735                 goto out_unlock;
 736         }
 737         if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
 738                 vma->vm_file)) {
 739                 r = -EPERM;
 740                 goto out_unlock;
 741         }
 742         mmap_read_unlock(mm);
 743         timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 744
 745 retry:
 746         range->notifier_seq = mmu_interval_read_begin(&bo->notifier);
 747
 748         mmap_read_lock(mm);
 749         r = hmm_range_fault(range);
 750         mmap_read_unlock(mm);
 751         if (unlikely(r)) {
 752                 /*
 753                  * FIXME: This timeout should encompass the retry from
 754                  * mmu_interval_read_retry() as well.
 755                  */
 756                 if (r == -EBUSY && !time_after(jiffies, timeout))
 757                         goto retry;
 758                 goto out_free_pfns;
 759         }
 760
 761         /*
 762          * Due to default_flags, all pages are HMM_PFN_VALID or
 763          * hmm_range_fault() fails. FIXME: The pages cannot be touched outside
 764          * the notifier_lock, and mmu_interval_read_retry() must be done first.
 765          */
 766         for (i = 0; i < ttm->num_pages; i++)
 767                 pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);
 768
 769         gtt->range = range;
 770         mmput(mm);
 771
 772         return 0;
 773
 774 out_unlock:
 775         mmap_read_unlock(mm);
 776 out_free_pfns:
 777         kvfree(range->hmm_pfns);
 778 out_free_ranges:
 779         kfree(range);
 780 out:
 781         mmput(mm);
 782         return r;
 783 }
 784
 785 /*
 786  * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
 787  * Check if the pages backing this ttm range have been invalidated
 788  *
 789  * Returns: true if pages are still valid
 790  */
 791 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
 792 {
 793         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 794         bool r = false;
 795
 796         if (!gtt || !gtt->userptr)
 797                 return false;
 798
 799         DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%x\n",
 800                 gtt->userptr, ttm->num_pages);
 801
 802         WARN_ONCE(!gtt->range || !gtt->range->hmm_pfns,
 803                 "No user pages to check\n");
 804
 805         if (gtt->range) {
 806                 /*
 807                  * FIXME: Must always hold notifier_lock for this, and must
 808                  * not ignore the return code.
 809                  */
 810                 r = mmu_interval_read_retry(gtt->range->notifier,
 811                                          gtt->range->notifier_seq);
 812                 kvfree(gtt->range->hmm_pfns);
 813                 kfree(gtt->range);
 814                 gtt->range = NULL;
 815         }
 816
 817         return !r;
 818 }
 819 #endif
 820
 821 /*
 822  * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
 823  *
 824  * Called by amdgpu_cs_list_validate(). This creates the page list
 825  * that backs user memory and will ultimately be mapped into the device
 826  * address space.
 827  */
 828 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
 829 {
 830         unsigned long i;
 831
 832         for (i = 0; i < ttm->num_pages; ++i)
 833                 ttm->pages[i] = pages ? pages[i] : NULL;
 834 }
 835
 836 /*
 837  * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
 838  *
 839  * Called by amdgpu_ttm_backend_bind()
 840  **/
 841 static int amdgpu_ttm_tt_pin_userptr(struct ttm_bo_device *bdev,
 842                                      struct ttm_tt *ttm)
 843 {
 844         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 845         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 846         int r;
 847
 848         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 849         enum dma_data_direction direction = write ?
 850                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 851
 852         /* Allocate an SG array and squash pages into it */
 853         r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
 854                                       ttm->num_pages << PAGE_SHIFT,
 855                                       GFP_KERNEL);
 856         if (r)
 857                 goto release_sg;
 858
 859         /* Map SG to device */
 860         r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
 861         if (r)
 862                 goto release_sg;
 863
 864         /* convert SG to linear array of pages and dma addresses */
 865         drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,
 866                                        ttm->num_pages);
 867
 868         return 0;
 869
 870 release_sg:
 871         kfree(ttm->sg);
 872         ttm->sg = NULL;
 873         return r;
 874 }
 875
 876 /*
 877  * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
 878  */
 879 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_bo_device *bdev,
 880                                         struct ttm_tt *ttm)
 881 {
 882         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 883         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 884
 885         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 886         enum dma_data_direction direction = write ?
 887                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 888
 889         /* double check that we don't free the table twice */
 890         if (!ttm->sg->sgl)
 891                 return;
 892
 893         /* unmap the pages mapped to the device */
 894         dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
 895         sg_free_table(ttm->sg);
 896
 897 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
 898         if (gtt->range) {
 899                 unsigned long i;
 900
 901                 for (i = 0; i < ttm->num_pages; i++) {
 902                         if (ttm->pages[i] !=
 903                             hmm_pfn_to_page(gtt->range->hmm_pfns[i]))
 904                                 break;
 905                 }
 906
 907                 WARN((i == ttm->num_pages), "Missing get_user_page_done\n");
 908         }
 909 #endif
 910 }
 911
 912 static int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
 913                                 struct ttm_buffer_object *tbo,
 914                                 uint64_t flags)
 915 {
 916         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
 917         struct ttm_tt *ttm = tbo->ttm;
 918         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 919         int r;
 920
 921         if (amdgpu_bo_encrypted(abo))
 922                 flags |= AMDGPU_PTE_TMZ;
 923
 924         if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) {
 925                 uint64_t page_idx = 1;
 926
 927                 r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
 928                                 ttm->pages, gtt->ttm.dma_address, flags);
 929                 if (r)
 930                         goto gart_bind_fail;
 931
 932                 /* The memory type of the first page defaults to UC. Now
 933                  * modify the memory type to NC from the second page of
 934                  * the BO onward.
 935                  */
 936                 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
 937                 flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC);
 938
 939                 r = amdgpu_gart_bind(adev,
 940                                 gtt->offset + (page_idx << PAGE_SHIFT),
 941                                 ttm->num_pages - page_idx,
 942                                 &ttm->pages[page_idx],
 943                                 &(gtt->ttm.dma_address[page_idx]), flags);
 944         } else {
 945                 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
 946                                      ttm->pages, gtt->ttm.dma_address, flags);
 947         }
 948
 949 gart_bind_fail:
 950         if (r)
 951                 DRM_ERROR("failed to bind %u pages at 0x%08llX\n",
 952                           ttm->num_pages, gtt->offset);
 953
 954         return r;
 955 }
 956
 957 /*
 958  * amdgpu_ttm_backend_bind - Bind GTT memory
 959  *
 960  * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
 961  * This handles binding GTT memory to the device address space.
 962  */
 963 static int amdgpu_ttm_backend_bind(struct ttm_bo_device *bdev,
 964                                    struct ttm_tt *ttm,
 965                                    struct ttm_resource *bo_mem)
 966 {
 967         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 968         struct amdgpu_ttm_tt *gtt = (void*)ttm;
 969         uint64_t flags;
 970         int r = 0;
 971
 972         if (!bo_mem)
 973                 return -EINVAL;
 974
 975         if (gtt->bound)
 976                 return 0;
 977
 978         if (gtt->userptr) {
 979                 r = amdgpu_ttm_tt_pin_userptr(bdev, ttm);
 980                 if (r) {
 981                         DRM_ERROR("failed to pin userptr\n");
 982                         return r;
 983                 }
 984         }
 985         if (!ttm->num_pages) {
 986                 WARN(1, "nothing to bind %u pages for mreg %p back %p!\n",
 987                      ttm->num_pages, bo_mem, ttm);
 988         }
 989
 990         if (bo_mem->mem_type == AMDGPU_PL_GDS ||
 991             bo_mem->mem_type == AMDGPU_PL_GWS ||
 992             bo_mem->mem_type == AMDGPU_PL_OA)
 993                 return -EINVAL;
 994
 995         if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
 996                 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
 997                 return 0;
 998         }
 999
1000         /* compute PTE flags relevant to this BO memory */
1001         flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1002
1003         /* bind pages into GART page tables */
1004         gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
1005         r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1006                 ttm->pages, gtt->ttm.dma_address, flags);
1007
1008         if (r)
1009                 DRM_ERROR("failed to bind %u pages at 0x%08llX\n",
1010                           ttm->num_pages, gtt->offset);
1011         gtt->bound = true;
1012         return r;
1013 }
1014
1015 /*
1016  * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
1017  * through AGP or GART aperture.
1018  *
1019  * If bo is accessible through AGP aperture, then use AGP aperture
1020  * to access bo; otherwise allocate logical space in GART aperture
1021  * and map bo to GART aperture.
1022  */
1023 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1024 {
1025         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1026         struct ttm_operation_ctx ctx = { false, false };
1027         struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
1028         struct ttm_resource tmp;
1029         struct ttm_placement placement;
1030         struct ttm_place placements;
1031         uint64_t addr, flags;
1032         int r;
1033
1034         if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1035                 return 0;
1036
1037         addr = amdgpu_gmc_agp_addr(bo);
1038         if (addr != AMDGPU_BO_INVALID_OFFSET) {
1039                 bo->mem.start = addr >> PAGE_SHIFT;
1040         } else {
1041
1042                 /* allocate GART space */
1043                 tmp = bo->mem;
1044                 tmp.mm_node = NULL;
1045                 placement.num_placement = 1;
1046                 placement.placement = &placements;
1047                 placement.num_busy_placement = 1;
1048                 placement.busy_placement = &placements;
1049                 placements.fpfn = 0;
1050                 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1051                 placements.mem_type = TTM_PL_TT;
1052                 placements.flags = bo->mem.placement;
1053
1054                 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1055                 if (unlikely(r))
1056                         return r;
1057
1058                 /* compute PTE flags for this buffer object */
1059                 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1060
1061                 /* Bind pages */
1062                 gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1063                 r = amdgpu_ttm_gart_bind(adev, bo, flags);
1064                 if (unlikely(r)) {
1065                         ttm_resource_free(bo, &tmp);
1066                         return r;
1067                 }
1068
1069                 ttm_resource_free(bo, &bo->mem);
1070                 bo->mem = tmp;
1071         }
1072
1073         return 0;
1074 }
1075
1076 /*
1077  * amdgpu_ttm_recover_gart - Rebind GTT pages
1078  *
1079  * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1080  * rebind GTT pages during a GPU reset.
1081  */
1082 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1083 {
1084         struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1085         uint64_t flags;
1086         int r;
1087
1088         if (!tbo->ttm)
1089                 return 0;
1090
1091         flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1092         r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1093
1094         return r;
1095 }
1096
1097 /*
1098  * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1099  *
1100  * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1101  * ttm_tt_destroy().
1102  */
1103 static void amdgpu_ttm_backend_unbind(struct ttm_bo_device *bdev,
1104                                       struct ttm_tt *ttm)
1105 {
1106         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
1107         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1108         int r;
1109
1110         if (!gtt->bound)
1111                 return;
1112
1113         /* if the pages have userptr pinning then clear that first */
1114         if (gtt->userptr)
1115                 amdgpu_ttm_tt_unpin_userptr(bdev, ttm);
1116
1117         if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1118                 return;
1119
1120         /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1121         r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1122         if (r)
1123                 DRM_ERROR("failed to unbind %u pages at 0x%08llX\n",
1124                           gtt->ttm.num_pages, gtt->offset);
1125         gtt->bound = false;
1126 }
1127
1128 static void amdgpu_ttm_backend_destroy(struct ttm_bo_device *bdev,
1129                                        struct ttm_tt *ttm)
1130 {
1131         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1132
1133         amdgpu_ttm_backend_unbind(bdev, ttm);
1134         ttm_tt_destroy_common(bdev, ttm);
1135         if (gtt->usertask)
1136                 put_task_struct(gtt->usertask);
1137
1138         ttm_tt_fini(&gtt->ttm);
1139         kfree(gtt);
1140 }
1141
1142 /**
1143  * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1144  *
1145  * @bo: The buffer object to create a GTT ttm_tt object around
1146  * @page_flags: Page flags to be added to the ttm_tt object
1147  *
1148  * Called by ttm_tt_create().
1149  */
1150 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1151                                            uint32_t page_flags)
1152 {
1153         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1154         struct amdgpu_ttm_tt *gtt;
1155         enum ttm_caching caching;
1156
1157         gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1158         if (gtt == NULL) {
1159                 return NULL;
1160         }
1161         gtt->gobj = &bo->base;
1162
1163         if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
1164                 caching = ttm_write_combined;
1165         else
1166                 caching = ttm_cached;
1167
1168         /* allocate space for the uninitialized page entries */
1169         if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags, caching)) {
1170                 kfree(gtt);
1171                 return NULL;
1172         }
1173         return &gtt->ttm;
1174 }
1175
1176 /*
1177  * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1178  *
1179  * Map the pages of a ttm_tt object to an address space visible
1180  * to the underlying device.
1181  */
1182 static int amdgpu_ttm_tt_populate(struct ttm_bo_device *bdev,
1183                                   struct ttm_tt *ttm,
1184                                   struct ttm_operation_ctx *ctx)
1185 {
1186         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
1187         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1188
1189         /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1190         if (gtt && gtt->userptr) {
1191                 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1192                 if (!ttm->sg)
1193                         return -ENOMEM;
1194
1195                 ttm->page_flags |= TTM_PAGE_FLAG_SG;
1196                 return 0;
1197         }
1198
1199         if (ttm->page_flags & TTM_PAGE_FLAG_SG) {
1200                 if (!ttm->sg) {
1201                         struct dma_buf_attachment *attach;
1202                         struct sg_table *sgt;
1203
1204                         attach = gtt->gobj->import_attach;
1205                         sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
1206                         if (IS_ERR(sgt))
1207                                 return PTR_ERR(sgt);
1208
1209                         ttm->sg = sgt;
1210                 }
1211
1212                 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,
1213                                                ttm->num_pages);
1214                 return 0;
1215         }
1216
1217         return ttm_pool_alloc(&adev->mman.bdev.pool, ttm, ctx);
1218 }
1219
1220 /*
1221  * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1222  *
1223  * Unmaps pages of a ttm_tt object from the device address space and
1224  * unpopulates the page array backing it.
1225  */
1226 static void amdgpu_ttm_tt_unpopulate(struct ttm_bo_device *bdev,
1227                                      struct ttm_tt *ttm)
1228 {
1229         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1230         struct amdgpu_device *adev;
1231
1232         if (gtt && gtt->userptr) {
1233                 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1234                 kfree(ttm->sg);
1235                 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1236                 return;
1237         }
1238
1239         if (ttm->sg && gtt->gobj->import_attach) {
1240                 struct dma_buf_attachment *attach;
1241
1242                 attach = gtt->gobj->import_attach;
1243                 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);
1244                 ttm->sg = NULL;
1245                 return;
1246         }
1247
1248         if (ttm->page_flags & TTM_PAGE_FLAG_SG)
1249                 return;
1250
1251         adev = amdgpu_ttm_adev(bdev);
1252         return ttm_pool_free(&adev->mman.bdev.pool, ttm);
1253 }
1254
1255 /**
1256  * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1257  * task
1258  *
1259  * @bo: The ttm_buffer_object to bind this userptr to
1260  * @addr:  The address in the current tasks VM space to use
1261  * @flags: Requirements of userptr object.
1262  *
1263  * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
1264  * to current task
1265  */
1266 int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,
1267                               uint64_t addr, uint32_t flags)
1268 {
1269         struct amdgpu_ttm_tt *gtt;
1270
1271         if (!bo->ttm) {
1272                 /* TODO: We want a separate TTM object type for userptrs */
1273                 bo->ttm = amdgpu_ttm_tt_create(bo, 0);
1274                 if (bo->ttm == NULL)
1275                         return -ENOMEM;
1276         }
1277
1278         gtt = (void *)bo->ttm;
1279         gtt->userptr = addr;
1280         gtt->userflags = flags;
1281
1282         if (gtt->usertask)
1283                 put_task_struct(gtt->usertask);
1284         gtt->usertask = current->group_leader;
1285         get_task_struct(gtt->usertask);
1286
1287         return 0;
1288 }
1289
1290 /*
1291  * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1292  */
1293 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1294 {
1295         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1296
1297         if (gtt == NULL)
1298                 return NULL;
1299
1300         if (gtt->usertask == NULL)
1301                 return NULL;
1302
1303         return gtt->usertask->mm;
1304 }
1305
1306 /*
1307  * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1308  * address range for the current task.
1309  *
1310  */
1311 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1312                                   unsigned long end)
1313 {
1314         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1315         unsigned long size;
1316
1317         if (gtt == NULL || !gtt->userptr)
1318                 return false;
1319
1320         /* Return false if no part of the ttm_tt object lies within
1321          * the range
1322          */
1323         size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE;
1324         if (gtt->userptr > end || gtt->userptr + size <= start)
1325                 return false;
1326
1327         return true;
1328 }
1329
1330 /*
1331  * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1332  */
1333 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1334 {
1335         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1336
1337         if (gtt == NULL || !gtt->userptr)
1338                 return false;
1339
1340         return true;
1341 }
1342
1343 /*
1344  * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1345  */
1346 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1347 {
1348         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1349
1350         if (gtt == NULL)
1351                 return false;
1352
1353         return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1354 }
1355
1356 /**
1357  * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1358  *
1359  * @ttm: The ttm_tt object to compute the flags for
1360  * @mem: The memory registry backing this ttm_tt object
1361  *
1362  * Figure out the flags to use for a VM PDE (Page Directory Entry).
1363  */
1364 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem)
1365 {
1366         uint64_t flags = 0;
1367
1368         if (mem && mem->mem_type != TTM_PL_SYSTEM)
1369                 flags |= AMDGPU_PTE_VALID;
1370
1371         if (mem && mem->mem_type == TTM_PL_TT) {
1372                 flags |= AMDGPU_PTE_SYSTEM;
1373
1374                 if (ttm->caching == ttm_cached)
1375                         flags |= AMDGPU_PTE_SNOOPED;
1376         }
1377
1378         if (mem && mem->mem_type == TTM_PL_VRAM &&
1379                         mem->bus.caching == ttm_cached)
1380                 flags |= AMDGPU_PTE_SNOOPED;
1381
1382         return flags;
1383 }
1384
1385 /**
1386  * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1387  *
1388  * @adev: amdgpu_device pointer
1389  * @ttm: The ttm_tt object to compute the flags for
1390  * @mem: The memory registry backing this ttm_tt object
1391  *
1392  * Figure out the flags to use for a VM PTE (Page Table Entry).
1393  */
1394 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1395                                  struct ttm_resource *mem)
1396 {
1397         uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1398
1399         flags |= adev->gart.gart_pte_flags;
1400         flags |= AMDGPU_PTE_READABLE;
1401
1402         if (!amdgpu_ttm_tt_is_readonly(ttm))
1403                 flags |= AMDGPU_PTE_WRITEABLE;
1404
1405         return flags;
1406 }
1407
1408 /*
1409  * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1410  * object.
1411  *
1412  * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1413  * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1414  * it can find space for a new object and by ttm_bo_force_list_clean() which is
1415  * used to clean out a memory space.
1416  */
1417 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1418                                             const struct ttm_place *place)
1419 {
1420         unsigned long num_pages = bo->mem.num_pages;
1421         struct drm_mm_node *node = bo->mem.mm_node;
1422         struct dma_resv_list *flist;
1423         struct dma_fence *f;
1424         int i;
1425
1426         if (bo->type == ttm_bo_type_kernel &&
1427             !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
1428                 return false;
1429
1430         /* If bo is a KFD BO, check if the bo belongs to the current process.
1431          * If true, then return false as any KFD process needs all its BOs to
1432          * be resident to run successfully
1433          */
1434         flist = dma_resv_get_list(bo->base.resv);
1435         if (flist) {
1436                 for (i = 0; i < flist->shared_count; ++i) {
1437                         f = rcu_dereference_protected(flist->shared[i],
1438                                 dma_resv_held(bo->base.resv));
1439                         if (amdkfd_fence_check_mm(f, current->mm))
1440                                 return false;
1441                 }
1442         }
1443
1444         switch (bo->mem.mem_type) {
1445         case TTM_PL_TT:
1446                 if (amdgpu_bo_is_amdgpu_bo(bo) &&
1447                     amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo)))
1448                         return false;
1449                 return true;
1450
1451         case TTM_PL_VRAM:
1452                 /* Check each drm MM node individually */
1453                 while (num_pages) {
1454                         if (place->fpfn < (node->start + node->size) &&
1455                             !(place->lpfn && place->lpfn <= node->start))
1456                                 return true;
1457
1458                         num_pages -= node->size;
1459                         ++node;
1460                 }
1461                 return false;
1462
1463         default:
1464                 break;
1465         }
1466
1467         return ttm_bo_eviction_valuable(bo, place);
1468 }
1469
1470 /**
1471  * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1472  *
1473  * @bo:  The buffer object to read/write
1474  * @offset:  Offset into buffer object
1475  * @buf:  Secondary buffer to write/read from
1476  * @len: Length in bytes of access
1477  * @write:  true if writing
1478  *
1479  * This is used to access VRAM that backs a buffer object via MMIO
1480  * access for debugging purposes.
1481  */
1482 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1483                                     unsigned long offset,
1484                                     void *buf, int len, int write)
1485 {
1486         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1487         struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1488         struct drm_mm_node *nodes;
1489         uint32_t value = 0;
1490         int ret = 0;
1491         uint64_t pos;
1492         unsigned long flags;
1493
1494         if (bo->mem.mem_type != TTM_PL_VRAM)
1495                 return -EIO;
1496
1497         pos = offset;
1498         nodes = amdgpu_find_mm_node(&abo->tbo.mem, &pos);
1499         pos += (nodes->start << PAGE_SHIFT);
1500
1501         while (len && pos < adev->gmc.mc_vram_size) {
1502                 uint64_t aligned_pos = pos & ~(uint64_t)3;
1503                 uint64_t bytes = 4 - (pos & 3);
1504                 uint32_t shift = (pos & 3) * 8;
1505                 uint32_t mask = 0xffffffff << shift;
1506
1507                 if (len < bytes) {
1508                         mask &= 0xffffffff >> (bytes - len) * 8;
1509                         bytes = len;
1510                 }
1511
1512                 if (mask != 0xffffffff) {
1513                         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1514                         WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1515                         WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1516                         if (!write || mask != 0xffffffff)
1517                                 value = RREG32_NO_KIQ(mmMM_DATA);
1518                         if (write) {
1519                                 value &= ~mask;
1520                                 value |= (*(uint32_t *)buf << shift) & mask;
1521                                 WREG32_NO_KIQ(mmMM_DATA, value);
1522                         }
1523                         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1524                         if (!write) {
1525                                 value = (value & mask) >> shift;
1526                                 memcpy(buf, &value, bytes);
1527                         }
1528                 } else {
1529                         bytes = (nodes->start + nodes->size) << PAGE_SHIFT;
1530                         bytes = min(bytes - pos, (uint64_t)len & ~0x3ull);
1531
1532                         amdgpu_device_vram_access(adev, pos, (uint32_t *)buf,
1533                                                   bytes, write);
1534                 }
1535
1536                 ret += bytes;
1537                 buf = (uint8_t *)buf + bytes;
1538                 pos += bytes;
1539                 len -= bytes;
1540                 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1541                         ++nodes;
1542                         pos = (nodes->start << PAGE_SHIFT);
1543                 }
1544         }
1545
1546         return ret;
1547 }
1548
1549 static void
1550 amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo)
1551 {
1552         amdgpu_bo_move_notify(bo, false, NULL);
1553 }
1554
1555 static struct ttm_bo_driver amdgpu_bo_driver = {
1556         .ttm_tt_create = &amdgpu_ttm_tt_create,
1557         .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1558         .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1559         .ttm_tt_destroy = &amdgpu_ttm_backend_destroy,
1560         .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1561         .evict_flags = &amdgpu_evict_flags,
1562         .move = &amdgpu_bo_move,
1563         .verify_access = &amdgpu_verify_access,
1564         .delete_mem_notify = &amdgpu_bo_delete_mem_notify,
1565         .release_notify = &amdgpu_bo_release_notify,
1566         .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1567         .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1568         .access_memory = &amdgpu_ttm_access_memory,
1569         .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
1570 };
1571
1572 /*
1573  * Firmware Reservation functions
1574  */
1575 /**
1576  * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1577  *
1578  * @adev: amdgpu_device pointer
1579  *
1580  * free fw reserved vram if it has been reserved.
1581  */
1582 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1583 {
1584         amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo,
1585                 NULL, &adev->mman.fw_vram_usage_va);
1586 }
1587
1588 /**
1589  * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1590  *
1591  * @adev: amdgpu_device pointer
1592  *
1593  * create bo vram reservation from fw.
1594  */
1595 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1596 {
1597         uint64_t vram_size = adev->gmc.visible_vram_size;
1598
1599         adev->mman.fw_vram_usage_va = NULL;
1600         adev->mman.fw_vram_usage_reserved_bo = NULL;
1601
1602         if (adev->mman.fw_vram_usage_size == 0 ||
1603             adev->mman.fw_vram_usage_size > vram_size)
1604                 return 0;
1605
1606         return amdgpu_bo_create_kernel_at(adev,
1607                                           adev->mman.fw_vram_usage_start_offset,
1608                                           adev->mman.fw_vram_usage_size,
1609                                           AMDGPU_GEM_DOMAIN_VRAM,
1610                                           &adev->mman.fw_vram_usage_reserved_bo,
1611                                           &adev->mman.fw_vram_usage_va);
1612 }
1613
1614 /*
1615  * Memoy training reservation functions
1616  */
1617
1618 /**
1619  * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1620  *
1621  * @adev: amdgpu_device pointer
1622  *
1623  * free memory training reserved vram if it has been reserved.
1624  */
1625 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev)
1626 {
1627         struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1628
1629         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
1630         amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL);
1631         ctx->c2p_bo = NULL;
1632
1633         return 0;
1634 }
1635
1636 static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev)
1637 {
1638         struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1639
1640         memset(ctx, 0, sizeof(*ctx));
1641
1642         ctx->c2p_train_data_offset =
1643                 ALIGN((adev->gmc.mc_vram_size - adev->mman.discovery_tmr_size - SZ_1M), SZ_1M);
1644         ctx->p2c_train_data_offset =
1645                 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET);
1646         ctx->train_data_size =
1647                 GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES;
1648
1649         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
1650                         ctx->train_data_size,
1651                         ctx->p2c_train_data_offset,
1652                         ctx->c2p_train_data_offset);
1653 }
1654
1655 /*
1656  * reserve TMR memory at the top of VRAM which holds
1657  * IP Discovery data and is protected by PSP.
1658  */
1659 static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev)
1660 {
1661         int ret;
1662         struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1663         bool mem_train_support = false;
1664
1665         if (!amdgpu_sriov_vf(adev)) {
1666                 ret = amdgpu_mem_train_support(adev);
1667                 if (ret == 1)
1668                         mem_train_support = true;
1669                 else if (ret == -1)
1670                         return -EINVAL;
1671                 else
1672                         DRM_DEBUG("memory training does not support!\n");
1673         }
1674
1675         /*
1676          * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all
1677          * the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc)
1678          *
1679          * Otherwise, fallback to legacy approach to check and reserve tmr block for ip
1680          * discovery data and G6 memory training data respectively
1681          */
1682         adev->mman.discovery_tmr_size =
1683                 amdgpu_atomfirmware_get_fw_reserved_fb_size(adev);
1684         if (!adev->mman.discovery_tmr_size)
1685                 adev->mman.discovery_tmr_size = DISCOVERY_TMR_OFFSET;
1686
1687         if (mem_train_support) {
1688                 /* reserve vram for mem train according to TMR location */
1689                 amdgpu_ttm_training_data_block_init(adev);
1690                 ret = amdgpu_bo_create_kernel_at(adev,
1691                                          ctx->c2p_train_data_offset,
1692                                          ctx->train_data_size,
1693                                          AMDGPU_GEM_DOMAIN_VRAM,
1694                                          &ctx->c2p_bo,
1695                                          NULL);
1696                 if (ret) {
1697                         DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret);
1698                         amdgpu_ttm_training_reserve_vram_fini(adev);
1699                         return ret;
1700                 }
1701                 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS;
1702         }
1703
1704         ret = amdgpu_bo_create_kernel_at(adev,
1705                                 adev->gmc.real_vram_size - adev->mman.discovery_tmr_size,
1706                                 adev->mman.discovery_tmr_size,
1707                                 AMDGPU_GEM_DOMAIN_VRAM,
1708                                 &adev->mman.discovery_memory,
1709                                 NULL);
1710         if (ret) {
1711                 DRM_ERROR("alloc tmr failed(%d)!\n", ret);
1712                 amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL);
1713                 return ret;
1714         }
1715
1716         return 0;
1717 }
1718
1719 /*
1720  * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1721  * gtt/vram related fields.
1722  *
1723  * This initializes all of the memory space pools that the TTM layer
1724  * will need such as the GTT space (system memory mapped to the device),
1725  * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1726  * can be mapped per VMID.
1727  */
1728 int amdgpu_ttm_init(struct amdgpu_device *adev)
1729 {
1730         uint64_t gtt_size;
1731         int r;
1732         u64 vis_vram_limit;
1733
1734         mutex_init(&adev->mman.gtt_window_lock);
1735
1736         /* No others user of address space so set it to 0 */
1737         r = ttm_bo_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev,
1738                                adev_to_drm(adev)->anon_inode->i_mapping,
1739                                adev_to_drm(adev)->vma_offset_manager,
1740                                adev->need_swiotlb,
1741                                dma_addressing_limited(adev->dev));
1742         if (r) {
1743                 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1744                 return r;
1745         }
1746         adev->mman.initialized = true;
1747
1748         /* Initialize VRAM pool with all of VRAM divided into pages */
1749         r = amdgpu_vram_mgr_init(adev);
1750         if (r) {
1751                 DRM_ERROR("Failed initializing VRAM heap.\n");
1752                 return r;
1753         }
1754
1755         /* Reduce size of CPU-visible VRAM if requested */
1756         vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1757         if (amdgpu_vis_vram_limit > 0 &&
1758             vis_vram_limit <= adev->gmc.visible_vram_size)
1759                 adev->gmc.visible_vram_size = vis_vram_limit;
1760
1761         /* Change the size here instead of the init above so only lpfn is affected */
1762         amdgpu_ttm_set_buffer_funcs_status(adev, false);
1763 #ifdef CONFIG_64BIT
1764 #ifdef CONFIG_X86
1765         if (adev->gmc.xgmi.connected_to_cpu)
1766                 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base,
1767                                 adev->gmc.visible_vram_size);
1768
1769         else
1770 #endif
1771                 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1772                                 adev->gmc.visible_vram_size);
1773 #endif
1774
1775         /*
1776          *The reserved vram for firmware must be pinned to the specified
1777          *place on the VRAM, so reserve it early.
1778          */
1779         r = amdgpu_ttm_fw_reserve_vram_init(adev);
1780         if (r) {
1781                 return r;
1782         }
1783
1784         /*
1785          * only NAVI10 and onwards ASIC support for IP discovery.
1786          * If IP discovery enabled, a block of memory should be
1787          * reserved for IP discovey.
1788          */
1789         if (adev->mman.discovery_bin) {
1790                 r = amdgpu_ttm_reserve_tmr(adev);
1791                 if (r)
1792                         return r;
1793         }
1794
1795         /* allocate memory as required for VGA
1796          * This is used for VGA emulation and pre-OS scanout buffers to
1797          * avoid display artifacts while transitioning between pre-OS
1798          * and driver.  */
1799         r = amdgpu_bo_create_kernel_at(adev, 0, adev->mman.stolen_vga_size,
1800                                        AMDGPU_GEM_DOMAIN_VRAM,
1801                                        &adev->mman.stolen_vga_memory,
1802                                        NULL);
1803         if (r)
1804                 return r;
1805         r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size,
1806                                        adev->mman.stolen_extended_size,
1807                                        AMDGPU_GEM_DOMAIN_VRAM,
1808                                        &adev->mman.stolen_extended_memory,
1809                                        NULL);
1810         if (r)
1811                 return r;
1812
1813         DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1814                  (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1815
1816         /* Compute GTT size, either bsaed on 3/4th the size of RAM size
1817          * or whatever the user passed on module init */
1818         if (amdgpu_gtt_size == -1) {
1819                 struct sysinfo si;
1820
1821                 si_meminfo(&si);
1822                 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1823                                adev->gmc.mc_vram_size),
1824                                ((uint64_t)si.totalram * si.mem_unit * 3/4));
1825         }
1826         else
1827                 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1828
1829         /* Initialize GTT memory pool */
1830         r = amdgpu_gtt_mgr_init(adev, gtt_size);
1831         if (r) {
1832                 DRM_ERROR("Failed initializing GTT heap.\n");
1833                 return r;
1834         }
1835         DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1836                  (unsigned)(gtt_size / (1024 * 1024)));
1837
1838         /* Initialize various on-chip memory pools */
1839         r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size);
1840         if (r) {
1841                 DRM_ERROR("Failed initializing GDS heap.\n");
1842                 return r;
1843         }
1844
1845         r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size);
1846         if (r) {
1847                 DRM_ERROR("Failed initializing gws heap.\n");
1848                 return r;
1849         }
1850
1851         r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size);
1852         if (r) {
1853                 DRM_ERROR("Failed initializing oa heap.\n");
1854                 return r;
1855         }
1856
1857         return 0;
1858 }
1859
1860 /*
1861  * amdgpu_ttm_fini - De-initialize the TTM memory pools
1862  */
1863 void amdgpu_ttm_fini(struct amdgpu_device *adev)
1864 {
1865         if (!adev->mman.initialized)
1866                 return;
1867
1868         amdgpu_ttm_training_reserve_vram_fini(adev);
1869         /* return the stolen vga memory back to VRAM */
1870         amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL);
1871         amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL);
1872         /* return the IP Discovery TMR memory back to VRAM */
1873         amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL);
1874         amdgpu_ttm_fw_reserve_vram_fini(adev);
1875
1876         if (adev->mman.aper_base_kaddr)
1877                 iounmap(adev->mman.aper_base_kaddr);
1878         adev->mman.aper_base_kaddr = NULL;
1879
1880         amdgpu_vram_mgr_fini(adev);
1881         amdgpu_gtt_mgr_fini(adev);
1882         ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);
1883         ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS);
1884         ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);
1885         ttm_bo_device_release(&adev->mman.bdev);
1886         adev->mman.initialized = false;
1887         DRM_INFO("amdgpu: ttm finalized\n");
1888 }
1889
1890 /**
1891  * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
1892  *
1893  * @adev: amdgpu_device pointer
1894  * @enable: true when we can use buffer functions.
1895  *
1896  * Enable/disable use of buffer functions during suspend/resume. This should
1897  * only be called at bootup or when userspace isn't running.
1898  */
1899 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1900 {
1901         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
1902         uint64_t size;
1903         int r;
1904
1905         if (!adev->mman.initialized || amdgpu_in_reset(adev) ||
1906             adev->mman.buffer_funcs_enabled == enable)
1907                 return;
1908
1909         if (enable) {
1910                 struct amdgpu_ring *ring;
1911                 struct drm_gpu_scheduler *sched;
1912
1913                 ring = adev->mman.buffer_funcs_ring;
1914                 sched = &ring->sched;
1915                 r = drm_sched_entity_init(&adev->mman.entity,
1916                                           DRM_SCHED_PRIORITY_KERNEL, &sched,
1917                                           1, NULL);
1918                 if (r) {
1919                         DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
1920                                   r);
1921                         return;
1922                 }
1923         } else {
1924                 drm_sched_entity_destroy(&adev->mman.entity);
1925                 dma_fence_put(man->move);
1926                 man->move = NULL;
1927         }
1928
1929         /* this just adjusts TTM size idea, which sets lpfn to the correct value */
1930         if (enable)
1931                 size = adev->gmc.real_vram_size;
1932         else
1933                 size = adev->gmc.visible_vram_size;
1934         man->size = size >> PAGE_SHIFT;
1935         adev->mman.buffer_funcs_enabled = enable;
1936 }
1937
1938 static vm_fault_t amdgpu_ttm_fault(struct vm_fault *vmf)
1939 {
1940         struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
1941         vm_fault_t ret;
1942
1943         ret = ttm_bo_vm_reserve(bo, vmf);
1944         if (ret)
1945                 return ret;
1946
1947         ret = amdgpu_bo_fault_reserve_notify(bo);
1948         if (ret)
1949                 goto unlock;
1950
1951         ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
1952                                        TTM_BO_VM_NUM_PREFAULT, 1);
1953         if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
1954                 return ret;
1955
1956 unlock:
1957         dma_resv_unlock(bo->base.resv);
1958         return ret;
1959 }
1960
1961 static struct vm_operations_struct amdgpu_ttm_vm_ops = {
1962         .fault = amdgpu_ttm_fault,
1963         .open = ttm_bo_vm_open,
1964         .close = ttm_bo_vm_close,
1965         .access = ttm_bo_vm_access
1966 };
1967
1968 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
1969 {
1970         struct drm_file *file_priv = filp->private_data;
1971         struct amdgpu_device *adev = drm_to_adev(file_priv->minor->dev);
1972         int r;
1973
1974         r = ttm_bo_mmap(filp, vma, &adev->mman.bdev);
1975         if (unlikely(r != 0))
1976                 return r;
1977
1978         vma->vm_ops = &amdgpu_ttm_vm_ops;
1979         return 0;
1980 }
1981
1982 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
1983                        uint64_t dst_offset, uint32_t byte_count,
1984                        struct dma_resv *resv,
1985                        struct dma_fence **fence, bool direct_submit,
1986                        bool vm_needs_flush, bool tmz)
1987 {
1988         enum amdgpu_ib_pool_type pool = direct_submit ? AMDGPU_IB_POOL_DIRECT :
1989                 AMDGPU_IB_POOL_DELAYED;
1990         struct amdgpu_device *adev = ring->adev;
1991         struct amdgpu_job *job;
1992
1993         uint32_t max_bytes;
1994         unsigned num_loops, num_dw;
1995         unsigned i;
1996         int r;
1997
1998         if (direct_submit && !ring->sched.ready) {
1999                 DRM_ERROR("Trying to move memory with ring turned off.\n");
2000                 return -EINVAL;
2001         }
2002
2003         max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2004         num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2005         num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
2006
2007         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, pool, &job);
2008         if (r)
2009                 return r;
2010
2011         if (vm_needs_flush) {
2012                 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ?
2013                                         adev->gmc.pdb0_bo : adev->gart.bo);
2014                 job->vm_needs_flush = true;
2015         }
2016         if (resv) {
2017                 r = amdgpu_sync_resv(adev, &job->sync, resv,
2018                                      AMDGPU_SYNC_ALWAYS,
2019                                      AMDGPU_FENCE_OWNER_UNDEFINED);
2020                 if (r) {
2021                         DRM_ERROR("sync failed (%d).\n", r);
2022                         goto error_free;
2023                 }
2024         }
2025
2026         for (i = 0; i < num_loops; i++) {
2027                 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2028
2029                 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2030                                         dst_offset, cur_size_in_bytes, tmz);
2031
2032                 src_offset += cur_size_in_bytes;
2033                 dst_offset += cur_size_in_bytes;
2034                 byte_count -= cur_size_in_bytes;
2035         }
2036
2037         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2038         WARN_ON(job->ibs[0].length_dw > num_dw);
2039         if (direct_submit)
2040                 r = amdgpu_job_submit_direct(job, ring, fence);
2041         else
2042                 r = amdgpu_job_submit(job, &adev->mman.entity,
2043                                       AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2044         if (r)
2045                 goto error_free;
2046
2047         return r;
2048
2049 error_free:
2050         amdgpu_job_free(job);
2051         DRM_ERROR("Error scheduling IBs (%d)\n", r);
2052         return r;
2053 }
2054
2055 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2056                        uint32_t src_data,
2057                        struct dma_resv *resv,
2058                        struct dma_fence **fence)
2059 {
2060         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2061         uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2062         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2063
2064         struct amdgpu_res_cursor cursor;
2065         unsigned int num_loops, num_dw;
2066         uint64_t num_bytes;
2067
2068         struct amdgpu_job *job;
2069         int r;
2070
2071         if (!adev->mman.buffer_funcs_enabled) {
2072                 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2073                 return -EINVAL;
2074         }
2075
2076         if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2077                 r = amdgpu_ttm_alloc_gart(&bo->tbo);
2078                 if (r)
2079                         return r;
2080         }
2081
2082         num_bytes = bo->tbo.mem.num_pages << PAGE_SHIFT;
2083         num_loops = 0;
2084
2085         amdgpu_res_first(&bo->tbo.mem, 0, num_bytes, &cursor);
2086         while (cursor.remaining) {
2087                 num_loops += DIV_ROUND_UP_ULL(cursor.size, max_bytes);
2088                 amdgpu_res_next(&cursor, cursor.size);
2089         }
2090         num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2091
2092         /* for IB padding */
2093         num_dw += 64;
2094
2095         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, AMDGPU_IB_POOL_DELAYED,
2096                                      &job);
2097         if (r)
2098                 return r;
2099
2100         if (resv) {
2101                 r = amdgpu_sync_resv(adev, &job->sync, resv,
2102                                      AMDGPU_SYNC_ALWAYS,
2103                                      AMDGPU_FENCE_OWNER_UNDEFINED);
2104                 if (r) {
2105                         DRM_ERROR("sync failed (%d).\n", r);
2106                         goto error_free;
2107                 }
2108         }
2109
2110         amdgpu_res_first(&bo->tbo.mem, 0, num_bytes, &cursor);
2111         while (cursor.remaining) {
2112                 uint32_t cur_size = min_t(uint64_t, cursor.size, max_bytes);
2113                 uint64_t dst_addr = cursor.start;
2114
2115                 dst_addr += amdgpu_ttm_domain_start(adev, bo->tbo.mem.mem_type);
2116                 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr,
2117                                         cur_size);
2118
2119                 amdgpu_res_next(&cursor, cur_size);
2120         }
2121
2122         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2123         WARN_ON(job->ibs[0].length_dw > num_dw);
2124         r = amdgpu_job_submit(job, &adev->mman.entity,
2125                               AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2126         if (r)
2127                 goto error_free;
2128
2129         return 0;
2130
2131 error_free:
2132         amdgpu_job_free(job);
2133         return r;
2134 }
2135
2136 #if defined(CONFIG_DEBUG_FS)
2137
2138 static int amdgpu_mm_vram_table_show(struct seq_file *m, void *unused)
2139 {
2140         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
2141         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev,
2142                                                             TTM_PL_VRAM);
2143         struct drm_printer p = drm_seq_file_printer(m);
2144
2145         man->func->debug(man, &p);
2146         return 0;
2147 }
2148
2149 static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)
2150 {
2151         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
2152
2153         return ttm_pool_debugfs(&adev->mman.bdev.pool, m);
2154 }
2155
2156 static int amdgpu_mm_tt_table_show(struct seq_file *m, void *unused)
2157 {
2158         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
2159         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev,
2160                                                             TTM_PL_TT);
2161         struct drm_printer p = drm_seq_file_printer(m);
2162
2163         man->func->debug(man, &p);
2164         return 0;
2165 }
2166
2167 static int amdgpu_mm_gds_table_show(struct seq_file *m, void *unused)
2168 {
2169         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
2170         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev,
2171                                                             AMDGPU_PL_GDS);
2172         struct drm_printer p = drm_seq_file_printer(m);
2173
2174         man->func->debug(man, &p);
2175         return 0;
2176 }
2177
2178 static int amdgpu_mm_gws_table_show(struct seq_file *m, void *unused)
2179 {
2180         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
2181         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev,
2182                                                             AMDGPU_PL_GWS);
2183         struct drm_printer p = drm_seq_file_printer(m);
2184
2185         man->func->debug(man, &p);
2186         return 0;
2187 }
2188
2189 static int amdgpu_mm_oa_table_show(struct seq_file *m, void *unused)
2190 {
2191         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
2192         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev,
2193                                                             AMDGPU_PL_OA);
2194         struct drm_printer p = drm_seq_file_printer(m);
2195
2196         man->func->debug(man, &p);
2197         return 0;
2198 }
2199
2200 DEFINE_SHOW_ATTRIBUTE(amdgpu_mm_vram_table);
2201 DEFINE_SHOW_ATTRIBUTE(amdgpu_mm_tt_table);
2202 DEFINE_SHOW_ATTRIBUTE(amdgpu_mm_gds_table);
2203 DEFINE_SHOW_ATTRIBUTE(amdgpu_mm_gws_table);
2204 DEFINE_SHOW_ATTRIBUTE(amdgpu_mm_oa_table);
2205 DEFINE_SHOW_ATTRIBUTE(amdgpu_ttm_page_pool);
2206
2207 /*
2208  * amdgpu_ttm_vram_read - Linear read access to VRAM
2209  *
2210  * Accesses VRAM via MMIO for debugging purposes.
2211  */
2212 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2213                                     size_t size, loff_t *pos)
2214 {
2215         struct amdgpu_device *adev = file_inode(f)->i_private;
2216         ssize_t result = 0;
2217
2218         if (size & 0x3 || *pos & 0x3)
2219                 return -EINVAL;
2220
2221         if (*pos >= adev->gmc.mc_vram_size)
2222                 return -ENXIO;
2223
2224         size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos));
2225         while (size) {
2226                 size_t bytes = min(size, AMDGPU_TTM_VRAM_MAX_DW_READ * 4);
2227                 uint32_t value[AMDGPU_TTM_VRAM_MAX_DW_READ];
2228
2229                 amdgpu_device_vram_access(adev, *pos, value, bytes, false);
2230                 if (copy_to_user(buf, value, bytes))
2231                         return -EFAULT;
2232
2233                 result += bytes;
2234                 buf += bytes;
2235                 *pos += bytes;
2236                 size -= bytes;
2237         }
2238
2239         return result;
2240 }
2241
2242 /*
2243  * amdgpu_ttm_vram_write - Linear write access to VRAM
2244  *
2245  * Accesses VRAM via MMIO for debugging purposes.
2246  */
2247 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2248                                     size_t size, loff_t *pos)
2249 {
2250         struct amdgpu_device *adev = file_inode(f)->i_private;
2251         ssize_t result = 0;
2252         int r;
2253
2254         if (size & 0x3 || *pos & 0x3)
2255                 return -EINVAL;
2256
2257         if (*pos >= adev->gmc.mc_vram_size)
2258                 return -ENXIO;
2259
2260         while (size) {
2261                 unsigned long flags;
2262                 uint32_t value;
2263
2264                 if (*pos >= adev->gmc.mc_vram_size)
2265                         return result;
2266
2267                 r = get_user(value, (uint32_t *)buf);
2268                 if (r)
2269                         return r;
2270
2271                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2272                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2273                 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2274                 WREG32_NO_KIQ(mmMM_DATA, value);
2275                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2276
2277                 result += 4;
2278                 buf += 4;
2279                 *pos += 4;
2280                 size -= 4;
2281         }
2282
2283         return result;
2284 }
2285
2286 static const struct file_operations amdgpu_ttm_vram_fops = {
2287         .owner = THIS_MODULE,
2288         .read = amdgpu_ttm_vram_read,
2289         .write = amdgpu_ttm_vram_write,
2290         .llseek = default_llseek,
2291 };
2292
2293 /*
2294  * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2295  *
2296  * This function is used to read memory that has been mapped to the
2297  * GPU and the known addresses are not physical addresses but instead
2298  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2299  */
2300 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2301                                  size_t size, loff_t *pos)
2302 {
2303         struct amdgpu_device *adev = file_inode(f)->i_private;
2304         struct iommu_domain *dom;
2305         ssize_t result = 0;
2306         int r;
2307
2308         /* retrieve the IOMMU domain if any for this device */
2309         dom = iommu_get_domain_for_dev(adev->dev);
2310
2311         while (size) {
2312                 phys_addr_t addr = *pos & PAGE_MASK;
2313                 loff_t off = *pos & ~PAGE_MASK;
2314                 size_t bytes = PAGE_SIZE - off;
2315                 unsigned long pfn;
2316                 struct page *p;
2317                 void *ptr;
2318
2319                 bytes = bytes < size ? bytes : size;
2320
2321                 /* Translate the bus address to a physical address.  If
2322                  * the domain is NULL it means there is no IOMMU active
2323                  * and the address translation is the identity
2324                  */
2325                 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2326
2327                 pfn = addr >> PAGE_SHIFT;
2328                 if (!pfn_valid(pfn))
2329                         return -EPERM;
2330
2331                 p = pfn_to_page(pfn);
2332                 if (p->mapping != adev->mman.bdev.dev_mapping)
2333                         return -EPERM;
2334
2335                 ptr = kmap(p);
2336                 r = copy_to_user(buf, ptr + off, bytes);
2337                 kunmap(p);
2338                 if (r)
2339                         return -EFAULT;
2340
2341                 size -= bytes;
2342                 *pos += bytes;
2343                 result += bytes;
2344         }
2345
2346         return result;
2347 }
2348
2349 /*
2350  * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2351  *
2352  * This function is used to write memory that has been mapped to the
2353  * GPU and the known addresses are not physical addresses but instead
2354  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2355  */
2356 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2357                                  size_t size, loff_t *pos)
2358 {
2359         struct amdgpu_device *adev = file_inode(f)->i_private;
2360         struct iommu_domain *dom;
2361         ssize_t result = 0;
2362         int r;
2363
2364         dom = iommu_get_domain_for_dev(adev->dev);
2365
2366         while (size) {
2367                 phys_addr_t addr = *pos & PAGE_MASK;
2368                 loff_t off = *pos & ~PAGE_MASK;
2369                 size_t bytes = PAGE_SIZE - off;
2370                 unsigned long pfn;
2371                 struct page *p;
2372                 void *ptr;
2373
2374                 bytes = bytes < size ? bytes : size;
2375
2376                 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2377
2378                 pfn = addr >> PAGE_SHIFT;
2379                 if (!pfn_valid(pfn))
2380                         return -EPERM;
2381
2382                 p = pfn_to_page(pfn);
2383                 if (p->mapping != adev->mman.bdev.dev_mapping)
2384                         return -EPERM;
2385
2386                 ptr = kmap(p);
2387                 r = copy_from_user(ptr + off, buf, bytes);
2388                 kunmap(p);
2389                 if (r)
2390                         return -EFAULT;
2391
2392                 size -= bytes;
2393                 *pos += bytes;
2394                 result += bytes;
2395         }
2396
2397         return result;
2398 }
2399
2400 static const struct file_operations amdgpu_ttm_iomem_fops = {
2401         .owner = THIS_MODULE,
2402         .read = amdgpu_iomem_read,
2403         .write = amdgpu_iomem_write,
2404         .llseek = default_llseek
2405 };
2406
2407 #endif
2408
2409 void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2410 {
2411 #if defined(CONFIG_DEBUG_FS)
2412         struct drm_minor *minor = adev_to_drm(adev)->primary;
2413         struct dentry *root = minor->debugfs_root;
2414
2415         debugfs_create_file_size("amdgpu_vram", 0444, root, adev,
2416                                  &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size);
2417         debugfs_create_file("amdgpu_iomem", 0444, root, adev,
2418                             &amdgpu_ttm_iomem_fops);
2419         debugfs_create_file("amdgpu_vram_mm", 0444, root, adev,
2420                             &amdgpu_mm_vram_table_fops);
2421         debugfs_create_file("amdgpu_gtt_mm", 0444, root, adev,
2422                             &amdgpu_mm_tt_table_fops);
2423         debugfs_create_file("amdgpu_gds_mm", 0444, root, adev,
2424                             &amdgpu_mm_gds_table_fops);
2425         debugfs_create_file("amdgpu_gws_mm", 0444, root, adev,
2426                             &amdgpu_mm_gws_table_fops);
2427         debugfs_create_file("amdgpu_oa_mm", 0444, root, adev,
2428                             &amdgpu_mm_oa_table_fops);
2429         debugfs_create_file("ttm_page_pool", 0444, root, adev,
2430                             &amdgpu_ttm_page_pool_fops);
2431 #endif
2432 }