drivers/gpu/drm/i915/gt/intel_ggtt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <linux/stop_machine.h>
   7
   8 #include <asm/set_memory.h>
   9 #include <asm/smp.h>
  10
  11 #include "intel_gt.h"
  12 #include "i915_drv.h"
  13 #include "i915_scatterlist.h"
  14 #include "i915_vgpu.h"
  15
  16 #include "intel_gtt.h"
  17
  18 static int
  19 i915_get_ggtt_vma_pages(struct i915_vma *vma);
  20
  21 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
  22                                    unsigned long color,
  23                                    u64 *start,
  24                                    u64 *end)
  25 {
  26         if (i915_node_color_differs(node, color))
  27                 *start += I915_GTT_PAGE_SIZE;
  28
  29         /*
  30          * Also leave a space between the unallocated reserved node after the
  31          * GTT and any objects within the GTT, i.e. we use the color adjustment
  32          * to insert a guard page to prevent prefetches crossing over the
  33          * GTT boundary.
  34          */
  35         node = list_next_entry(node, node_list);
  36         if (node->color != color)
  37                 *end -= I915_GTT_PAGE_SIZE;
  38 }
  39
  40 static int ggtt_init_hw(struct i915_ggtt *ggtt)
  41 {
  42         struct drm_i915_private *i915 = ggtt->vm.i915;
  43
  44         i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
  45
  46         ggtt->vm.is_ggtt = true;
  47
  48         /* Only VLV supports read-only GGTT mappings */
  49         ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
  50
  51         if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
  52                 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
  53
  54         if (ggtt->mappable_end) {
  55                 if (!io_mapping_init_wc(&ggtt->iomap,
  56                                         ggtt->gmadr.start,
  57                                         ggtt->mappable_end)) {
  58                         ggtt->vm.cleanup(&ggtt->vm);
  59                         return -EIO;
  60                 }
  61
  62                 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
  63                                               ggtt->mappable_end);
  64         }
  65
  66         i915_ggtt_init_fences(ggtt);
  67
  68         return 0;
  69 }
  70
  71 /**
  72  * i915_ggtt_init_hw - Initialize GGTT hardware
  73  * @i915: i915 device
  74  */
  75 int i915_ggtt_init_hw(struct drm_i915_private *i915)
  76 {
  77         int ret;
  78
  79         stash_init(&i915->mm.wc_stash);
  80
  81         /*
  82          * Note that we use page colouring to enforce a guard page at the
  83          * end of the address space. This is required as the CS may prefetch
  84          * beyond the end of the batch buffer, across the page boundary,
  85          * and beyond the end of the GTT if we do not provide a guard.
  86          */
  87         ret = ggtt_init_hw(&i915->ggtt);
  88         if (ret)
  89                 return ret;
  90
  91         return 0;
  92 }
  93
  94 /*
  95  * Certain Gen5 chipsets require require idling the GPU before
  96  * unmapping anything from the GTT when VT-d is enabled.
  97  */
  98 static bool needs_idle_maps(struct drm_i915_private *i915)
  99 {
 100         /*
 101          * Query intel_iommu to see if we need the workaround. Presumably that
 102          * was loaded first.
 103          */
 104         return IS_GEN(i915, 5) && IS_MOBILE(i915) && intel_vtd_active();
 105 }
 106
 107 void i915_ggtt_suspend(struct i915_ggtt *ggtt)
 108 {
 109         struct i915_vma *vma;
 110
 111         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 112                 i915_vma_wait_for_bind(vma);
 113
 114         ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
 115         ggtt->invalidate(ggtt);
 116
 117         intel_gt_check_and_clear_faults(ggtt->vm.gt);
 118 }
 119
 120 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
 121 {
 122         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 123
 124         spin_lock_irq(&uncore->lock);
 125         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 126         intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
 127         spin_unlock_irq(&uncore->lock);
 128 }
 129
 130 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
 131 {
 132         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 133
 134         /*
 135          * Note that as an uncached mmio write, this will flush the
 136          * WCB of the writes into the GGTT before it triggers the invalidate.
 137          */
 138         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 139 }
 140
 141 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 142 {
 143         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 144         struct drm_i915_private *i915 = ggtt->vm.i915;
 145
 146         gen8_ggtt_invalidate(ggtt);
 147
 148         if (INTEL_GEN(i915) >= 12)
 149                 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,
 150                                       GEN12_GUC_TLB_INV_CR_INVALIDATE);
 151         else
 152                 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 153 }
 154
 155 static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt)
 156 {
 157         intel_gtt_chipset_flush();
 158 }
 159
 160 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 161 {
 162         writeq(pte, addr);
 163 }
 164
 165 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 166                                   dma_addr_t addr,
 167                                   u64 offset,
 168                                   enum i915_cache_level level,
 169                                   u32 unused)
 170 {
 171         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 172         gen8_pte_t __iomem *pte =
 173                 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 174
 175         gen8_set_pte(pte, gen8_pte_encode(addr, level, 0));
 176
 177         ggtt->invalidate(ggtt);
 178 }
 179
 180 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 181                                      struct i915_vma *vma,
 182                                      enum i915_cache_level level,
 183                                      u32 flags)
 184 {
 185         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 186         struct sgt_iter sgt_iter;
 187         gen8_pte_t __iomem *gtt_entries;
 188         const gen8_pte_t pte_encode = gen8_pte_encode(0, level, 0);
 189         dma_addr_t addr;
 190
 191         /*
 192          * Note that we ignore PTE_READ_ONLY here. The caller must be careful
 193          * not to allow the user to override access to a read only page.
 194          */
 195
 196         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
 197         gtt_entries += vma->node.start / I915_GTT_PAGE_SIZE;
 198         for_each_sgt_daddr(addr, sgt_iter, vma->pages)
 199                 gen8_set_pte(gtt_entries++, pte_encode | addr);
 200
 201         /*
 202          * We want to flush the TLBs only after we're certain all the PTE
 203          * updates have finished.
 204          */
 205         ggtt->invalidate(ggtt);
 206 }
 207
 208 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 209                                   dma_addr_t addr,
 210                                   u64 offset,
 211                                   enum i915_cache_level level,
 212                                   u32 flags)
 213 {
 214         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 215         gen6_pte_t __iomem *pte =
 216                 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 217
 218         iowrite32(vm->pte_encode(addr, level, flags), pte);
 219
 220         ggtt->invalidate(ggtt);
 221 }
 222
 223 /*
 224  * Binds an object into the global gtt with the specified cache level.
 225  * The object will be accessible to the GPU via commands whose operands
 226  * reference offsets within the global GTT as well as accessible by the GPU
 227  * through the GMADR mapped BAR (i915->mm.gtt->gtt).
 228  */
 229 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 230                                      struct i915_vma *vma,
 231                                      enum i915_cache_level level,
 232                                      u32 flags)
 233 {
 234         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 235         gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
 236         unsigned int i = vma->node.start / I915_GTT_PAGE_SIZE;
 237         struct sgt_iter iter;
 238         dma_addr_t addr;
 239
 240         for_each_sgt_daddr(addr, iter, vma->pages)
 241                 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
 242
 243         /*
 244          * We want to flush the TLBs only after we're certain all the PTE
 245          * updates have finished.
 246          */
 247         ggtt->invalidate(ggtt);
 248 }
 249
 250 static void nop_clear_range(struct i915_address_space *vm,
 251                             u64 start, u64 length)
 252 {
 253 }
 254
 255 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 256                                   u64 start, u64 length)
 257 {
 258         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 259         unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 260         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 261         const gen8_pte_t scratch_pte = vm->scratch[0].encode;
 262         gen8_pte_t __iomem *gtt_base =
 263                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
 264         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 265         int i;
 266
 267         if (WARN(num_entries > max_entries,
 268                  "First entry = %d; Num entries = %d (max=%d)\n",
 269                  first_entry, num_entries, max_entries))
 270                 num_entries = max_entries;
 271
 272         for (i = 0; i < num_entries; i++)
 273                 gen8_set_pte(&gtt_base[i], scratch_pte);
 274 }
 275
 276 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
 277 {
 278         /*
 279          * Make sure the internal GAM fifo has been cleared of all GTT
 280          * writes before exiting stop_machine(). This guarantees that
 281          * any aperture accesses waiting to start in another process
 282          * cannot back up behind the GTT writes causing a hang.
 283          * The register can be any arbitrary GAM register.
 284          */
 285         intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
 286 }
 287
 288 struct insert_page {
 289         struct i915_address_space *vm;
 290         dma_addr_t addr;
 291         u64 offset;
 292         enum i915_cache_level level;
 293 };
 294
 295 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
 296 {
 297         struct insert_page *arg = _arg;
 298
 299         gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
 300         bxt_vtd_ggtt_wa(arg->vm);
 301
 302         return 0;
 303 }
 304
 305 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
 306                                           dma_addr_t addr,
 307                                           u64 offset,
 308                                           enum i915_cache_level level,
 309                                           u32 unused)
 310 {
 311         struct insert_page arg = { vm, addr, offset, level };
 312
 313         stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
 314 }
 315
 316 struct insert_entries {
 317         struct i915_address_space *vm;
 318         struct i915_vma *vma;
 319         enum i915_cache_level level;
 320         u32 flags;
 321 };
 322
 323 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 324 {
 325         struct insert_entries *arg = _arg;
 326
 327         gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags);
 328         bxt_vtd_ggtt_wa(arg->vm);
 329
 330         return 0;
 331 }
 332
 333 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
 334                                              struct i915_vma *vma,
 335                                              enum i915_cache_level level,
 336                                              u32 flags)
 337 {
 338         struct insert_entries arg = { vm, vma, level, flags };
 339
 340         stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
 341 }
 342
 343 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 344                                   u64 start, u64 length)
 345 {
 346         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 347         unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 348         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 349         gen6_pte_t scratch_pte, __iomem *gtt_base =
 350                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
 351         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 352         int i;
 353
 354         if (WARN(num_entries > max_entries,
 355                  "First entry = %d; Num entries = %d (max=%d)\n",
 356                  first_entry, num_entries, max_entries))
 357                 num_entries = max_entries;
 358
 359         scratch_pte = vm->scratch[0].encode;
 360         for (i = 0; i < num_entries; i++)
 361                 iowrite32(scratch_pte, &gtt_base[i]);
 362 }
 363
 364 static void i915_ggtt_insert_page(struct i915_address_space *vm,
 365                                   dma_addr_t addr,
 366                                   u64 offset,
 367                                   enum i915_cache_level cache_level,
 368                                   u32 unused)
 369 {
 370         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 371                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 372
 373         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
 374 }
 375
 376 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 377                                      struct i915_vma *vma,
 378                                      enum i915_cache_level cache_level,
 379                                      u32 unused)
 380 {
 381         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 382                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 383
 384         intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
 385                                     flags);
 386 }
 387
 388 static void i915_ggtt_clear_range(struct i915_address_space *vm,
 389                                   u64 start, u64 length)
 390 {
 391         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
 392 }
 393
 394 static int ggtt_bind_vma(struct i915_vma *vma,
 395                          enum i915_cache_level cache_level,
 396                          u32 flags)
 397 {
 398         struct drm_i915_gem_object *obj = vma->obj;
 399         u32 pte_flags;
 400
 401         /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 402         pte_flags = 0;
 403         if (i915_gem_object_is_readonly(obj))
 404                 pte_flags |= PTE_READ_ONLY;
 405
 406         vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
 407
 408         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
 409
 410         /*
 411          * Without aliasing PPGTT there's no difference between
 412          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
 413          * upgrade to both bound if we bind either to avoid double-binding.
 414          */
 415         atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
 416
 417         return 0;
 418 }
 419
 420 static void ggtt_unbind_vma(struct i915_vma *vma)
 421 {
 422         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
 423 }
 424
 425 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
 426 {
 427         u64 size;
 428         int ret;
 429
 430         if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
 431                 return 0;
 432
 433         GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
 434         size = ggtt->vm.total - GUC_GGTT_TOP;
 435
 436         ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size,
 437                                    GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
 438                                    PIN_NOEVICT);
 439         if (ret)
 440                 drm_dbg(&ggtt->vm.i915->drm,
 441                         "Failed to reserve top of GGTT for GuC\n");
 442
 443         return ret;
 444 }
 445
 446 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
 447 {
 448         if (drm_mm_node_allocated(&ggtt->uc_fw))
 449                 drm_mm_remove_node(&ggtt->uc_fw);
 450 }
 451
 452 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
 453 {
 454         ggtt_release_guc_top(ggtt);
 455         if (drm_mm_node_allocated(&ggtt->error_capture))
 456                 drm_mm_remove_node(&ggtt->error_capture);
 457         mutex_destroy(&ggtt->error_mutex);
 458 }
 459
 460 static int init_ggtt(struct i915_ggtt *ggtt)
 461 {
 462         /*
 463          * Let GEM Manage all of the aperture.
 464          *
 465          * However, leave one page at the end still bound to the scratch page.
 466          * There are a number of places where the hardware apparently prefetches
 467          * past the end of the object, and we've seen multiple hangs with the
 468          * GPU head pointer stuck in a batchbuffer bound at the last page of the
 469          * aperture.  One page should be enough to keep any prefetching inside
 470          * of the aperture.
 471          */
 472         unsigned long hole_start, hole_end;
 473         struct drm_mm_node *entry;
 474         int ret;
 475
 476         /*
 477          * GuC requires all resources that we're sharing with it to be placed in
 478          * non-WOPCM memory. If GuC is not present or not in use we still need a
 479          * small bias as ring wraparound at offset 0 sometimes hangs. No idea
 480          * why.
 481          */
 482         ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
 483                                intel_wopcm_guc_size(&ggtt->vm.i915->wopcm));
 484
 485         ret = intel_vgt_balloon(ggtt);
 486         if (ret)
 487                 return ret;
 488
 489         mutex_init(&ggtt->error_mutex);
 490         if (ggtt->mappable_end) {
 491                 /* Reserve a mappable slot for our lockless error capture */
 492                 ret = drm_mm_insert_node_in_range(&ggtt->vm.mm,
 493                                                   &ggtt->error_capture,
 494                                                   PAGE_SIZE, 0,
 495                                                   I915_COLOR_UNEVICTABLE,
 496                                                   0, ggtt->mappable_end,
 497                                                   DRM_MM_INSERT_LOW);
 498                 if (ret)
 499                         return ret;
 500         }
 501
 502         /*
 503          * The upper portion of the GuC address space has a sizeable hole
 504          * (several MB) that is inaccessible by GuC. Reserve this range within
 505          * GGTT as it can comfortably hold GuC/HuC firmware images.
 506          */
 507         ret = ggtt_reserve_guc_top(ggtt);
 508         if (ret)
 509                 goto err;
 510
 511         /* Clear any non-preallocated blocks */
 512         drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
 513                 drm_dbg_kms(&ggtt->vm.i915->drm,
 514                             "clearing unused GTT space: [%lx, %lx]\n",
 515                             hole_start, hole_end);
 516                 ggtt->vm.clear_range(&ggtt->vm, hole_start,
 517                                      hole_end - hole_start);
 518         }
 519
 520         /* And finally clear the reserved guard page */
 521         ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
 522
 523         return 0;
 524
 525 err:
 526         cleanup_init_ggtt(ggtt);
 527         return ret;
 528 }
 529
 530 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 531                                  enum i915_cache_level cache_level,
 532                                  u32 flags)
 533 {
 534         u32 pte_flags;
 535         int ret;
 536
 537         /* Currently applicable only to VLV */
 538         pte_flags = 0;
 539         if (i915_gem_object_is_readonly(vma->obj))
 540                 pte_flags |= PTE_READ_ONLY;
 541
 542         if (flags & I915_VMA_LOCAL_BIND) {
 543                 struct i915_ppgtt *alias = i915_vm_to_ggtt(vma->vm)->alias;
 544
 545                 if (flags & I915_VMA_ALLOC) {
 546                         ret = alias->vm.allocate_va_range(&alias->vm,
 547                                                           vma->node.start,
 548                                                           vma->size);
 549                         if (ret)
 550                                 return ret;
 551
 552                         set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma));
 553                 }
 554
 555                 GEM_BUG_ON(!test_bit(I915_VMA_ALLOC_BIT,
 556                                      __i915_vma_flags(vma)));
 557                 alias->vm.insert_entries(&alias->vm, vma,
 558                                          cache_level, pte_flags);
 559         }
 560
 561         if (flags & I915_VMA_GLOBAL_BIND)
 562                 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
 563
 564         return 0;
 565 }
 566
 567 static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
 568 {
 569         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
 570                 struct i915_address_space *vm = vma->vm;
 571
 572                 vm->clear_range(vm, vma->node.start, vma->size);
 573         }
 574
 575         if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) {
 576                 struct i915_address_space *vm =
 577                         &i915_vm_to_ggtt(vma->vm)->alias->vm;
 578
 579                 vm->clear_range(vm, vma->node.start, vma->size);
 580         }
 581 }
 582
 583 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
 584 {
 585         struct i915_ppgtt *ppgtt;
 586         int err;
 587
 588         ppgtt = i915_ppgtt_create(ggtt->vm.gt);
 589         if (IS_ERR(ppgtt))
 590                 return PTR_ERR(ppgtt);
 591
 592         if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
 593                 err = -ENODEV;
 594                 goto err_ppgtt;
 595         }
 596
 597         /*
 598          * Note we only pre-allocate as far as the end of the global
 599          * GTT. On 48b / 4-level page-tables, the difference is very,
 600          * very significant! We have to preallocate as GVT/vgpu does
 601          * not like the page directory disappearing.
 602          */
 603         err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total);
 604         if (err)
 605                 goto err_ppgtt;
 606
 607         ggtt->alias = ppgtt;
 608         ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
 609
 610         GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma);
 611         ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
 612
 613         GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma);
 614         ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
 615
 616         return 0;
 617
 618 err_ppgtt:
 619         i915_vm_put(&ppgtt->vm);
 620         return err;
 621 }
 622
 623 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
 624 {
 625         struct i915_ppgtt *ppgtt;
 626
 627         ppgtt = fetch_and_zero(&ggtt->alias);
 628         if (!ppgtt)
 629                 return;
 630
 631         i915_vm_put(&ppgtt->vm);
 632
 633         ggtt->vm.vma_ops.bind_vma   = ggtt_bind_vma;
 634         ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
 635 }
 636
 637 int i915_init_ggtt(struct drm_i915_private *i915)
 638 {
 639         int ret;
 640
 641         ret = init_ggtt(&i915->ggtt);
 642         if (ret)
 643                 return ret;
 644
 645         if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
 646                 ret = init_aliasing_ppgtt(&i915->ggtt);
 647                 if (ret)
 648                         cleanup_init_ggtt(&i915->ggtt);
 649         }
 650
 651         return 0;
 652 }
 653
 654 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
 655 {
 656         struct i915_vma *vma, *vn;
 657
 658         atomic_set(&ggtt->vm.open, 0);
 659
 660         rcu_barrier(); /* flush the RCU'ed__i915_vm_release */
 661         flush_workqueue(ggtt->vm.i915->wq);
 662
 663         mutex_lock(&ggtt->vm.mutex);
 664
 665         list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
 666                 WARN_ON(__i915_vma_unbind(vma));
 667
 668         if (drm_mm_node_allocated(&ggtt->error_capture))
 669                 drm_mm_remove_node(&ggtt->error_capture);
 670         mutex_destroy(&ggtt->error_mutex);
 671
 672         ggtt_release_guc_top(ggtt);
 673         intel_vgt_deballoon(ggtt);
 674
 675         ggtt->vm.cleanup(&ggtt->vm);
 676
 677         mutex_unlock(&ggtt->vm.mutex);
 678         i915_address_space_fini(&ggtt->vm);
 679
 680         arch_phys_wc_del(ggtt->mtrr);
 681
 682         if (ggtt->iomap.size)
 683                 io_mapping_fini(&ggtt->iomap);
 684 }
 685
 686 /**
 687  * i915_ggtt_driver_release - Clean up GGTT hardware initialization
 688  * @i915: i915 device
 689  */
 690 void i915_ggtt_driver_release(struct drm_i915_private *i915)
 691 {
 692         struct pagevec *pvec;
 693
 694         fini_aliasing_ppgtt(&i915->ggtt);
 695
 696         ggtt_cleanup_hw(&i915->ggtt);
 697
 698         pvec = &i915->mm.wc_stash.pvec;
 699         if (pvec->nr) {
 700                 set_pages_array_wb(pvec->pages, pvec->nr);
 701                 __pagevec_release(pvec);
 702         }
 703 }
 704
 705 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
 706 {
 707         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
 708         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
 709         return snb_gmch_ctl << 20;
 710 }
 711
 712 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
 713 {
 714         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
 715         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
 716         if (bdw_gmch_ctl)
 717                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
 718
 719 #ifdef CONFIG_X86_32
 720         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
 721         if (bdw_gmch_ctl > 4)
 722                 bdw_gmch_ctl = 4;
 723 #endif
 724
 725         return bdw_gmch_ctl << 20;
 726 }
 727
 728 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
 729 {
 730         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
 731         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
 732
 733         if (gmch_ctrl)
 734                 return 1 << (20 + gmch_ctrl);
 735
 736         return 0;
 737 }
 738
 739 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 740 {
 741         struct drm_i915_private *i915 = ggtt->vm.i915;
 742         struct pci_dev *pdev = i915->drm.pdev;
 743         phys_addr_t phys_addr;
 744         int ret;
 745
 746         /* For Modern GENs the PTEs and register space are split in the BAR */
 747         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
 748
 749         /*
 750          * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
 751          * will be dropped. For WC mappings in general we have 64 byte burst
 752          * writes when the WC buffer is flushed, so we can't use it, but have to
 753          * resort to an uncached mapping. The WC issue is easily caught by the
 754          * readback check when writing GTT PTE entries.
 755          */
 756         if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10)
 757                 ggtt->gsm = ioremap(phys_addr, size);
 758         else
 759                 ggtt->gsm = ioremap_wc(phys_addr, size);
 760         if (!ggtt->gsm) {
 761                 DRM_ERROR("Failed to map the ggtt page table\n");
 762                 return -ENOMEM;
 763         }
 764
 765         ret = setup_scratch_page(&ggtt->vm, GFP_DMA32);
 766         if (ret) {
 767                 DRM_ERROR("Scratch setup failed\n");
 768                 /* iounmap will also get called at remove, but meh */
 769                 iounmap(ggtt->gsm);
 770                 return ret;
 771         }
 772
 773         ggtt->vm.scratch[0].encode =
 774                 ggtt->vm.pte_encode(px_dma(&ggtt->vm.scratch[0]),
 775                                     I915_CACHE_NONE, 0);
 776
 777         return 0;
 778 }
 779
 780 int ggtt_set_pages(struct i915_vma *vma)
 781 {
 782         int ret;
 783
 784         GEM_BUG_ON(vma->pages);
 785
 786         ret = i915_get_ggtt_vma_pages(vma);
 787         if (ret)
 788                 return ret;
 789
 790         vma->page_sizes = vma->obj->mm.page_sizes;
 791
 792         return 0;
 793 }
 794
 795 static void gen6_gmch_remove(struct i915_address_space *vm)
 796 {
 797         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 798
 799         iounmap(ggtt->gsm);
 800         cleanup_scratch_page(vm);
 801 }
 802
 803 static struct resource pci_resource(struct pci_dev *pdev, int bar)
 804 {
 805         return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar),
 806                                                pci_resource_len(pdev, bar));
 807 }
 808
 809 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 810 {
 811         struct drm_i915_private *i915 = ggtt->vm.i915;
 812         struct pci_dev *pdev = i915->drm.pdev;
 813         unsigned int size;
 814         u16 snb_gmch_ctl;
 815         int err;
 816
 817         /* TODO: We're not aware of mappable constraints on gen8 yet */
 818         if (!IS_DGFX(i915)) {
 819                 ggtt->gmadr = pci_resource(pdev, 2);
 820                 ggtt->mappable_end = resource_size(&ggtt->gmadr);
 821         }
 822
 823         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
 824         if (!err)
 825                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
 826         if (err)
 827                 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
 828
 829         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 830         if (IS_CHERRYVIEW(i915))
 831                 size = chv_get_total_gtt_size(snb_gmch_ctl);
 832         else
 833                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
 834
 835         ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
 836         ggtt->vm.cleanup = gen6_gmch_remove;
 837         ggtt->vm.insert_page = gen8_ggtt_insert_page;
 838         ggtt->vm.clear_range = nop_clear_range;
 839         if (intel_scanout_needs_vtd_wa(i915))
 840                 ggtt->vm.clear_range = gen8_ggtt_clear_range;
 841
 842         ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
 843
 844         /* Serialize GTT updates with aperture access on BXT if VT-d is on. */
 845         if (intel_ggtt_update_needs_vtd_wa(i915) ||
 846             IS_CHERRYVIEW(i915) /* fails with concurrent use/update */) {
 847                 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
 848                 ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
 849                 ggtt->vm.bind_async_flags =
 850                         I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 851         }
 852
 853         ggtt->invalidate = gen8_ggtt_invalidate;
 854
 855         ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
 856         ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
 857         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
 858         ggtt->vm.vma_ops.clear_pages = clear_pages;
 859
 860         ggtt->vm.pte_encode = gen8_pte_encode;
 861
 862         setup_private_pat(ggtt->vm.gt->uncore);
 863
 864         return ggtt_probe_common(ggtt, size);
 865 }
 866
 867 static u64 snb_pte_encode(dma_addr_t addr,
 868                           enum i915_cache_level level,
 869                           u32 flags)
 870 {
 871         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 872
 873         switch (level) {
 874         case I915_CACHE_L3_LLC:
 875         case I915_CACHE_LLC:
 876                 pte |= GEN6_PTE_CACHE_LLC;
 877                 break;
 878         case I915_CACHE_NONE:
 879                 pte |= GEN6_PTE_UNCACHED;
 880                 break;
 881         default:
 882                 MISSING_CASE(level);
 883         }
 884
 885         return pte;
 886 }
 887
 888 static u64 ivb_pte_encode(dma_addr_t addr,
 889                           enum i915_cache_level level,
 890                           u32 flags)
 891 {
 892         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 893
 894         switch (level) {
 895         case I915_CACHE_L3_LLC:
 896                 pte |= GEN7_PTE_CACHE_L3_LLC;
 897                 break;
 898         case I915_CACHE_LLC:
 899                 pte |= GEN6_PTE_CACHE_LLC;
 900                 break;
 901         case I915_CACHE_NONE:
 902                 pte |= GEN6_PTE_UNCACHED;
 903                 break;
 904         default:
 905                 MISSING_CASE(level);
 906         }
 907
 908         return pte;
 909 }
 910
 911 static u64 byt_pte_encode(dma_addr_t addr,
 912                           enum i915_cache_level level,
 913                           u32 flags)
 914 {
 915         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 916
 917         if (!(flags & PTE_READ_ONLY))
 918                 pte |= BYT_PTE_WRITEABLE;
 919
 920         if (level != I915_CACHE_NONE)
 921                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 922
 923         return pte;
 924 }
 925
 926 static u64 hsw_pte_encode(dma_addr_t addr,
 927                           enum i915_cache_level level,
 928                           u32 flags)
 929 {
 930         gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 931
 932         if (level != I915_CACHE_NONE)
 933                 pte |= HSW_WB_LLC_AGE3;
 934
 935         return pte;
 936 }
 937
 938 static u64 iris_pte_encode(dma_addr_t addr,
 939                            enum i915_cache_level level,
 940                            u32 flags)
 941 {
 942         gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 943
 944         switch (level) {
 945         case I915_CACHE_NONE:
 946                 break;
 947         case I915_CACHE_WT:
 948                 pte |= HSW_WT_ELLC_LLC_AGE3;
 949                 break;
 950         default:
 951                 pte |= HSW_WB_ELLC_LLC_AGE3;
 952                 break;
 953         }
 954
 955         return pte;
 956 }
 957
 958 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 959 {
 960         struct drm_i915_private *i915 = ggtt->vm.i915;
 961         struct pci_dev *pdev = i915->drm.pdev;
 962         unsigned int size;
 963         u16 snb_gmch_ctl;
 964         int err;
 965
 966         ggtt->gmadr = pci_resource(pdev, 2);
 967         ggtt->mappable_end = resource_size(&ggtt->gmadr);
 968
 969         /*
 970          * 64/512MB is the current min/max we actually know of, but this is
 971          * just a coarse sanity check.
 972          */
 973         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
 974                 DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end);
 975                 return -ENXIO;
 976         }
 977
 978         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40));
 979         if (!err)
 980                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
 981         if (err)
 982                 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
 983         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 984
 985         size = gen6_get_total_gtt_size(snb_gmch_ctl);
 986         ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
 987
 988         ggtt->vm.clear_range = nop_clear_range;
 989         if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915))
 990                 ggtt->vm.clear_range = gen6_ggtt_clear_range;
 991         ggtt->vm.insert_page = gen6_ggtt_insert_page;
 992         ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
 993         ggtt->vm.cleanup = gen6_gmch_remove;
 994
 995         ggtt->invalidate = gen6_ggtt_invalidate;
 996
 997         if (HAS_EDRAM(i915))
 998                 ggtt->vm.pte_encode = iris_pte_encode;
 999         else if (IS_HASWELL(i915))
1000                 ggtt->vm.pte_encode = hsw_pte_encode;
1001         else if (IS_VALLEYVIEW(i915))
1002                 ggtt->vm.pte_encode = byt_pte_encode;
1003         else if (INTEL_GEN(i915) >= 7)
1004                 ggtt->vm.pte_encode = ivb_pte_encode;
1005         else
1006                 ggtt->vm.pte_encode = snb_pte_encode;
1007
1008         ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
1009         ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
1010         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
1011         ggtt->vm.vma_ops.clear_pages = clear_pages;
1012
1013         return ggtt_probe_common(ggtt, size);
1014 }
1015
1016 static void i915_gmch_remove(struct i915_address_space *vm)
1017 {
1018         intel_gmch_remove();
1019 }
1020
1021 static int i915_gmch_probe(struct i915_ggtt *ggtt)
1022 {
1023         struct drm_i915_private *i915 = ggtt->vm.i915;
1024         phys_addr_t gmadr_base;
1025         int ret;
1026
1027         ret = intel_gmch_probe(i915->bridge_dev, i915->drm.pdev, NULL);
1028         if (!ret) {
1029                 DRM_ERROR("failed to set up gmch\n");
1030                 return -EIO;
1031         }
1032
1033         intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end);
1034
1035         ggtt->gmadr =
1036                 (struct resource)DEFINE_RES_MEM(gmadr_base, ggtt->mappable_end);
1037
1038         ggtt->do_idle_maps = needs_idle_maps(i915);
1039         ggtt->vm.insert_page = i915_ggtt_insert_page;
1040         ggtt->vm.insert_entries = i915_ggtt_insert_entries;
1041         ggtt->vm.clear_range = i915_ggtt_clear_range;
1042         ggtt->vm.cleanup = i915_gmch_remove;
1043
1044         ggtt->invalidate = gmch_ggtt_invalidate;
1045
1046         ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
1047         ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
1048         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
1049         ggtt->vm.vma_ops.clear_pages = clear_pages;
1050
1051         if (unlikely(ggtt->do_idle_maps))
1052                 dev_notice(i915->drm.dev,
1053                            "Applying Ironlake quirks for intel_iommu\n");
1054
1055         return 0;
1056 }
1057
1058 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1059 {
1060         struct drm_i915_private *i915 = gt->i915;
1061         int ret;
1062
1063         ggtt->vm.gt = gt;
1064         ggtt->vm.i915 = i915;
1065         ggtt->vm.dma = &i915->drm.pdev->dev;
1066
1067         if (INTEL_GEN(i915) <= 5)
1068                 ret = i915_gmch_probe(ggtt);
1069         else if (INTEL_GEN(i915) < 8)
1070                 ret = gen6_gmch_probe(ggtt);
1071         else
1072                 ret = gen8_gmch_probe(ggtt);
1073         if (ret)
1074                 return ret;
1075
1076         if ((ggtt->vm.total - 1) >> 32) {
1077                 DRM_ERROR("We never expected a Global GTT with more than 32bits"
1078                           " of address space! Found %lldM!\n",
1079                           ggtt->vm.total >> 20);
1080                 ggtt->vm.total = 1ULL << 32;
1081                 ggtt->mappable_end =
1082                         min_t(u64, ggtt->mappable_end, ggtt->vm.total);
1083         }
1084
1085         if (ggtt->mappable_end > ggtt->vm.total) {
1086                 DRM_ERROR("mappable aperture extends past end of GGTT,"
1087                           " aperture=%pa, total=%llx\n",
1088                           &ggtt->mappable_end, ggtt->vm.total);
1089                 ggtt->mappable_end = ggtt->vm.total;
1090         }
1091
1092         /* GMADR is the PCI mmio aperture into the global GTT. */
1093         DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->vm.total >> 20);
1094         DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20);
1095         DRM_DEBUG_DRIVER("DSM size = %lluM\n",
1096                          (u64)resource_size(&intel_graphics_stolen_res) >> 20);
1097
1098         return 0;
1099 }
1100
1101 /**
1102  * i915_ggtt_probe_hw - Probe GGTT hardware location
1103  * @i915: i915 device
1104  */
1105 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
1106 {
1107         int ret;
1108
1109         ret = ggtt_probe_hw(&i915->ggtt, &i915->gt);
1110         if (ret)
1111                 return ret;
1112
1113         if (intel_vtd_active())
1114                 dev_info(i915->drm.dev, "VT-d active for gfx access\n");
1115
1116         return 0;
1117 }
1118
1119 int i915_ggtt_enable_hw(struct drm_i915_private *i915)
1120 {
1121         if (INTEL_GEN(i915) < 6 && !intel_enable_gtt())
1122                 return -EIO;
1123
1124         return 0;
1125 }
1126
1127 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt)
1128 {
1129         GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate);
1130
1131         ggtt->invalidate = guc_ggtt_invalidate;
1132
1133         ggtt->invalidate(ggtt);
1134 }
1135
1136 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt)
1137 {
1138         /* XXX Temporary pardon for error unload */
1139         if (ggtt->invalidate == gen8_ggtt_invalidate)
1140                 return;
1141
1142         /* We should only be called after i915_ggtt_enable_guc() */
1143         GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate);
1144
1145         ggtt->invalidate = gen8_ggtt_invalidate;
1146
1147         ggtt->invalidate(ggtt);
1148 }
1149
1150 void i915_ggtt_resume(struct i915_ggtt *ggtt)
1151 {
1152         struct i915_vma *vma;
1153         bool flush = false;
1154         int open;
1155
1156         intel_gt_check_and_clear_faults(ggtt->vm.gt);
1157
1158         /* First fill our portion of the GTT with scratch pages */
1159         ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
1160
1161         /* Skip rewriting PTE on VMA unbind. */
1162         open = atomic_xchg(&ggtt->vm.open, 0);
1163
1164         /* clflush objects bound into the GGTT and rebind them. */
1165         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) {
1166                 struct drm_i915_gem_object *obj = vma->obj;
1167
1168                 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
1169                         continue;
1170
1171                 clear_bit(I915_VMA_GLOBAL_BIND_BIT, __i915_vma_flags(vma));
1172                 WARN_ON(i915_vma_bind(vma,
1173                                       obj ? obj->cache_level : 0,
1174                                       PIN_GLOBAL, NULL));
1175                 if (obj) { /* only used during resume => exclusive access */
1176                         flush |= fetch_and_zero(&obj->write_domain);
1177                         obj->read_domains |= I915_GEM_DOMAIN_GTT;
1178                 }
1179         }
1180
1181         atomic_set(&ggtt->vm.open, open);
1182         ggtt->invalidate(ggtt);
1183
1184         if (flush)
1185                 wbinvd_on_all_cpus();
1186
1187         if (INTEL_GEN(ggtt->vm.i915) >= 8)
1188                 setup_private_pat(ggtt->vm.gt->uncore);
1189 }
1190
1191 static struct scatterlist *
1192 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
1193              unsigned int width, unsigned int height,
1194              unsigned int stride,
1195              struct sg_table *st, struct scatterlist *sg)
1196 {
1197         unsigned int column, row;
1198         unsigned int src_idx;
1199
1200         for (column = 0; column < width; column++) {
1201                 src_idx = stride * (height - 1) + column + offset;
1202                 for (row = 0; row < height; row++) {
1203                         st->nents++;
1204                         /*
1205                          * We don't need the pages, but need to initialize
1206                          * the entries so the sg list can be happily traversed.
1207                          * The only thing we need are DMA addresses.
1208                          */
1209                         sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
1210                         sg_dma_address(sg) =
1211                                 i915_gem_object_get_dma_address(obj, src_idx);
1212                         sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
1213                         sg = sg_next(sg);
1214                         src_idx -= stride;
1215                 }
1216         }
1217
1218         return sg;
1219 }
1220
1221 static noinline struct sg_table *
1222 intel_rotate_pages(struct intel_rotation_info *rot_info,
1223                    struct drm_i915_gem_object *obj)
1224 {
1225         unsigned int size = intel_rotation_info_size(rot_info);
1226         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1227         struct sg_table *st;
1228         struct scatterlist *sg;
1229         int ret = -ENOMEM;
1230         int i;
1231
1232         /* Allocate target SG list. */
1233         st = kmalloc(sizeof(*st), GFP_KERNEL);
1234         if (!st)
1235                 goto err_st_alloc;
1236
1237         ret = sg_alloc_table(st, size, GFP_KERNEL);
1238         if (ret)
1239                 goto err_sg_alloc;
1240
1241         st->nents = 0;
1242         sg = st->sgl;
1243
1244         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
1245                 sg = rotate_pages(obj, rot_info->plane[i].offset,
1246                                   rot_info->plane[i].width, rot_info->plane[i].height,
1247                                   rot_info->plane[i].stride, st, sg);
1248         }
1249
1250         return st;
1251
1252 err_sg_alloc:
1253         kfree(st);
1254 err_st_alloc:
1255
1256         drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1257                 obj->base.size, rot_info->plane[0].width,
1258                 rot_info->plane[0].height, size);
1259
1260         return ERR_PTR(ret);
1261 }
1262
1263 static struct scatterlist *
1264 remap_pages(struct drm_i915_gem_object *obj, unsigned int offset,
1265             unsigned int width, unsigned int height,
1266             unsigned int stride,
1267             struct sg_table *st, struct scatterlist *sg)
1268 {
1269         unsigned int row;
1270
1271         for (row = 0; row < height; row++) {
1272                 unsigned int left = width * I915_GTT_PAGE_SIZE;
1273
1274                 while (left) {
1275                         dma_addr_t addr;
1276                         unsigned int length;
1277
1278                         /*
1279                          * We don't need the pages, but need to initialize
1280                          * the entries so the sg list can be happily traversed.
1281                          * The only thing we need are DMA addresses.
1282                          */
1283
1284                         addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1285
1286                         length = min(left, length);
1287
1288                         st->nents++;
1289
1290                         sg_set_page(sg, NULL, length, 0);
1291                         sg_dma_address(sg) = addr;
1292                         sg_dma_len(sg) = length;
1293                         sg = sg_next(sg);
1294
1295                         offset += length / I915_GTT_PAGE_SIZE;
1296                         left -= length;
1297                 }
1298
1299                 offset += stride - width;
1300         }
1301
1302         return sg;
1303 }
1304
1305 static noinline struct sg_table *
1306 intel_remap_pages(struct intel_remapped_info *rem_info,
1307                   struct drm_i915_gem_object *obj)
1308 {
1309         unsigned int size = intel_remapped_info_size(rem_info);
1310         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1311         struct sg_table *st;
1312         struct scatterlist *sg;
1313         int ret = -ENOMEM;
1314         int i;
1315
1316         /* Allocate target SG list. */
1317         st = kmalloc(sizeof(*st), GFP_KERNEL);
1318         if (!st)
1319                 goto err_st_alloc;
1320
1321         ret = sg_alloc_table(st, size, GFP_KERNEL);
1322         if (ret)
1323                 goto err_sg_alloc;
1324
1325         st->nents = 0;
1326         sg = st->sgl;
1327
1328         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
1329                 sg = remap_pages(obj, rem_info->plane[i].offset,
1330                                  rem_info->plane[i].width, rem_info->plane[i].height,
1331                                  rem_info->plane[i].stride, st, sg);
1332         }
1333
1334         i915_sg_trim(st);
1335
1336         return st;
1337
1338 err_sg_alloc:
1339         kfree(st);
1340 err_st_alloc:
1341
1342         drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1343                 obj->base.size, rem_info->plane[0].width,
1344                 rem_info->plane[0].height, size);
1345
1346         return ERR_PTR(ret);
1347 }
1348
1349 static noinline struct sg_table *
1350 intel_partial_pages(const struct i915_ggtt_view *view,
1351                     struct drm_i915_gem_object *obj)
1352 {
1353         struct sg_table *st;
1354         struct scatterlist *sg, *iter;
1355         unsigned int count = view->partial.size;
1356         unsigned int offset;
1357         int ret = -ENOMEM;
1358
1359         st = kmalloc(sizeof(*st), GFP_KERNEL);
1360         if (!st)
1361                 goto err_st_alloc;
1362
1363         ret = sg_alloc_table(st, count, GFP_KERNEL);
1364         if (ret)
1365                 goto err_sg_alloc;
1366
1367         iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
1368         GEM_BUG_ON(!iter);
1369
1370         sg = st->sgl;
1371         st->nents = 0;
1372         do {
1373                 unsigned int len;
1374
1375                 len = min(iter->length - (offset << PAGE_SHIFT),
1376                           count << PAGE_SHIFT);
1377                 sg_set_page(sg, NULL, len, 0);
1378                 sg_dma_address(sg) =
1379                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
1380                 sg_dma_len(sg) = len;
1381
1382                 st->nents++;
1383                 count -= len >> PAGE_SHIFT;
1384                 if (count == 0) {
1385                         sg_mark_end(sg);
1386                         i915_sg_trim(st); /* Drop any unused tail entries. */
1387
1388                         return st;
1389                 }
1390
1391                 sg = __sg_next(sg);
1392                 iter = __sg_next(iter);
1393                 offset = 0;
1394         } while (1);
1395
1396 err_sg_alloc:
1397         kfree(st);
1398 err_st_alloc:
1399         return ERR_PTR(ret);
1400 }
1401
1402 static int
1403 i915_get_ggtt_vma_pages(struct i915_vma *vma)
1404 {
1405         int ret;
1406
1407         /*
1408          * The vma->pages are only valid within the lifespan of the borrowed
1409          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1410          * must be the vma->pages. A simple rule is that vma->pages must only
1411          * be accessed when the obj->mm.pages are pinned.
1412          */
1413         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1414
1415         switch (vma->ggtt_view.type) {
1416         default:
1417                 GEM_BUG_ON(vma->ggtt_view.type);
1418                 /* fall through */
1419         case I915_GGTT_VIEW_NORMAL:
1420                 vma->pages = vma->obj->mm.pages;
1421                 return 0;
1422
1423         case I915_GGTT_VIEW_ROTATED:
1424                 vma->pages =
1425                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
1426                 break;
1427
1428         case I915_GGTT_VIEW_REMAPPED:
1429                 vma->pages =
1430                         intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
1431                 break;
1432
1433         case I915_GGTT_VIEW_PARTIAL:
1434                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
1435                 break;
1436         }
1437
1438         ret = 0;
1439         if (IS_ERR(vma->pages)) {
1440                 ret = PTR_ERR(vma->pages);
1441                 vma->pages = NULL;
1442                 drm_err(&vma->vm->i915->drm,
1443                         "Failed to get pages for VMA view type %u (%d)!\n",
1444                         vma->ggtt_view.type, ret);
1445         }
1446         return ret;
1447 }