drivers/gpu/drm/i915/gt/intel_ppgtt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <linux/slab.h>
   7
   8 #include "gem/i915_gem_lmem.h"
   9
  10 #include "i915_trace.h"
  11 #include "intel_gtt.h"
  12 #include "gen6_ppgtt.h"
  13 #include "gen8_ppgtt.h"
  14
  15 struct i915_page_table *alloc_pt(struct i915_address_space *vm, int sz)
  16 {
  17         struct i915_page_table *pt;
  18
  19         pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL);
  20         if (unlikely(!pt))
  21                 return ERR_PTR(-ENOMEM);
  22
  23         pt->base = vm->alloc_pt_dma(vm, sz);
  24         if (IS_ERR(pt->base)) {
  25                 kfree(pt);
  26                 return ERR_PTR(-ENOMEM);
  27         }
  28
  29         pt->is_compact = false;
  30         atomic_set(&pt->used, 0);
  31         return pt;
  32 }
  33
  34 struct i915_page_directory *__alloc_pd(int count)
  35 {
  36         struct i915_page_directory *pd;
  37
  38         pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
  39         if (unlikely(!pd))
  40                 return NULL;
  41
  42         pd->entry = kcalloc(count, sizeof(*pd->entry), I915_GFP_ALLOW_FAIL);
  43         if (unlikely(!pd->entry)) {
  44                 kfree(pd);
  45                 return NULL;
  46         }
  47
  48         spin_lock_init(&pd->lock);
  49         return pd;
  50 }
  51
  52 struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
  53 {
  54         struct i915_page_directory *pd;
  55
  56         pd = __alloc_pd(I915_PDES);
  57         if (unlikely(!pd))
  58                 return ERR_PTR(-ENOMEM);
  59
  60         pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);
  61         if (IS_ERR(pd->pt.base)) {
  62                 kfree(pd->entry);
  63                 kfree(pd);
  64                 return ERR_PTR(-ENOMEM);
  65         }
  66
  67         return pd;
  68 }
  69
  70 void free_px(struct i915_address_space *vm, struct i915_page_table *pt, int lvl)
  71 {
  72         BUILD_BUG_ON(offsetof(struct i915_page_directory, pt));
  73
  74         if (lvl) {
  75                 struct i915_page_directory *pd =
  76                         container_of(pt, typeof(*pd), pt);
  77                 kfree(pd->entry);
  78         }
  79
  80         if (pt->base)
  81                 i915_gem_object_put(pt->base);
  82
  83         kfree(pt);
  84 }
  85
  86 static void
  87 write_dma_entry(struct drm_i915_gem_object * const pdma,
  88                 const unsigned short idx,
  89                 const u64 encoded_entry)
  90 {
  91         u64 * const vaddr = __px_vaddr(pdma);
  92
  93         vaddr[idx] = encoded_entry;
  94         drm_clflush_virt_range(&vaddr[idx], sizeof(u64));
  95 }
  96
  97 void
  98 __set_pd_entry(struct i915_page_directory * const pd,
  99                const unsigned short idx,
 100                struct i915_page_table * const to,
 101                u64 (*encode)(const dma_addr_t, const enum i915_cache_level))
 102 {
 103         /* Each thread pre-pins the pd, and we may have a thread per pde. */
 104         GEM_BUG_ON(atomic_read(px_used(pd)) > NALLOC * I915_PDES);
 105
 106         atomic_inc(px_used(pd));
 107         pd->entry[idx] = to;
 108         write_dma_entry(px_base(pd), idx, encode(px_dma(to), I915_CACHE_LLC));
 109 }
 110
 111 void
 112 clear_pd_entry(struct i915_page_directory * const pd,
 113                const unsigned short idx,
 114                const struct drm_i915_gem_object * const scratch)
 115 {
 116         GEM_BUG_ON(atomic_read(px_used(pd)) == 0);
 117
 118         write_dma_entry(px_base(pd), idx, scratch->encode);
 119         pd->entry[idx] = NULL;
 120         atomic_dec(px_used(pd));
 121 }
 122
 123 bool
 124 release_pd_entry(struct i915_page_directory * const pd,
 125                  const unsigned short idx,
 126                  struct i915_page_table * const pt,
 127                  const struct drm_i915_gem_object * const scratch)
 128 {
 129         bool free = false;
 130
 131         if (atomic_add_unless(&pt->used, -1, 1))
 132                 return false;
 133
 134         spin_lock(&pd->lock);
 135         if (atomic_dec_and_test(&pt->used)) {
 136                 clear_pd_entry(pd, idx, scratch);
 137                 free = true;
 138         }
 139         spin_unlock(&pd->lock);
 140
 141         return free;
 142 }
 143
 144 int i915_ppgtt_init_hw(struct intel_gt *gt)
 145 {
 146         struct drm_i915_private *i915 = gt->i915;
 147
 148         gtt_write_workarounds(gt);
 149
 150         if (GRAPHICS_VER(i915) == 6)
 151                 gen6_ppgtt_enable(gt);
 152         else if (GRAPHICS_VER(i915) == 7)
 153                 gen7_ppgtt_enable(gt);
 154
 155         return 0;
 156 }
 157
 158 static struct i915_ppgtt *
 159 __ppgtt_create(struct intel_gt *gt, unsigned long lmem_pt_obj_flags)
 160 {
 161         if (GRAPHICS_VER(gt->i915) < 8)
 162                 return gen6_ppgtt_create(gt);
 163         else
 164                 return gen8_ppgtt_create(gt, lmem_pt_obj_flags);
 165 }
 166
 167 struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt,
 168                                      unsigned long lmem_pt_obj_flags)
 169 {
 170         struct i915_ppgtt *ppgtt;
 171
 172         ppgtt = __ppgtt_create(gt, lmem_pt_obj_flags);
 173         if (IS_ERR(ppgtt))
 174                 return ppgtt;
 175
 176         trace_i915_ppgtt_create(&ppgtt->vm);
 177
 178         return ppgtt;
 179 }
 180
 181 void ppgtt_bind_vma(struct i915_address_space *vm,
 182                     struct i915_vm_pt_stash *stash,
 183                     struct i915_vma_resource *vma_res,
 184                     enum i915_cache_level cache_level,
 185                     u32 flags)
 186 {
 187         u32 pte_flags;
 188
 189         if (!vma_res->allocated) {
 190                 vm->allocate_va_range(vm, stash, vma_res->start,
 191                                       vma_res->vma_size);
 192                 vma_res->allocated = true;
 193         }
 194
 195         /* Applicable to VLV, and gen8+ */
 196         pte_flags = 0;
 197         if (vma_res->bi.readonly)
 198                 pte_flags |= PTE_READ_ONLY;
 199         if (vma_res->bi.lmem)
 200                 pte_flags |= PTE_LM;
 201
 202         vm->insert_entries(vm, vma_res, cache_level, pte_flags);
 203         wmb();
 204 }
 205
 206 void ppgtt_unbind_vma(struct i915_address_space *vm,
 207                       struct i915_vma_resource *vma_res)
 208 {
 209         if (!vma_res->allocated)
 210                 return;
 211
 212         vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 213         if (vma_res->tlb)
 214                 vma_invalidate_tlb(vm, vma_res->tlb);
 215 }
 216
 217 static unsigned long pd_count(u64 size, int shift)
 218 {
 219         /* Beware later misalignment */
 220         return (size + 2 * (BIT_ULL(shift) - 1)) >> shift;
 221 }
 222
 223 int i915_vm_alloc_pt_stash(struct i915_address_space *vm,
 224                            struct i915_vm_pt_stash *stash,
 225                            u64 size)
 226 {
 227         unsigned long count;
 228         int shift, n, pt_sz;
 229
 230         shift = vm->pd_shift;
 231         if (!shift)
 232                 return 0;
 233
 234         pt_sz = stash->pt_sz;
 235         if (!pt_sz)
 236                 pt_sz = I915_GTT_PAGE_SIZE_4K;
 237         else
 238                 GEM_BUG_ON(!IS_DGFX(vm->i915));
 239
 240         GEM_BUG_ON(!is_power_of_2(pt_sz));
 241
 242         count = pd_count(size, shift);
 243         while (count--) {
 244                 struct i915_page_table *pt;
 245
 246                 pt = alloc_pt(vm, pt_sz);
 247                 if (IS_ERR(pt)) {
 248                         i915_vm_free_pt_stash(vm, stash);
 249                         return PTR_ERR(pt);
 250                 }
 251
 252                 pt->stash = stash->pt[0];
 253                 stash->pt[0] = pt;
 254         }
 255
 256         for (n = 1; n < vm->top; n++) {
 257                 shift += ilog2(I915_PDES); /* Each PD holds 512 entries */
 258                 count = pd_count(size, shift);
 259                 while (count--) {
 260                         struct i915_page_directory *pd;
 261
 262                         pd = alloc_pd(vm);
 263                         if (IS_ERR(pd)) {
 264                                 i915_vm_free_pt_stash(vm, stash);
 265                                 return PTR_ERR(pd);
 266                         }
 267
 268                         pd->pt.stash = stash->pt[1];
 269                         stash->pt[1] = &pd->pt;
 270                 }
 271         }
 272
 273         return 0;
 274 }
 275
 276 int i915_vm_map_pt_stash(struct i915_address_space *vm,
 277                          struct i915_vm_pt_stash *stash)
 278 {
 279         struct i915_page_table *pt;
 280         int n, err;
 281
 282         for (n = 0; n < ARRAY_SIZE(stash->pt); n++) {
 283                 for (pt = stash->pt[n]; pt; pt = pt->stash) {
 284                         err = map_pt_dma_locked(vm, pt->base);
 285                         if (err)
 286                                 return err;
 287                 }
 288         }
 289
 290         return 0;
 291 }
 292
 293 void i915_vm_free_pt_stash(struct i915_address_space *vm,
 294                            struct i915_vm_pt_stash *stash)
 295 {
 296         struct i915_page_table *pt;
 297         int n;
 298
 299         for (n = 0; n < ARRAY_SIZE(stash->pt); n++) {
 300                 while ((pt = stash->pt[n])) {
 301                         stash->pt[n] = pt->stash;
 302                         free_px(vm, pt, n);
 303                 }
 304         }
 305 }
 306
 307 void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt,
 308                 unsigned long lmem_pt_obj_flags)
 309 {
 310         struct drm_i915_private *i915 = gt->i915;
 311
 312         ppgtt->vm.gt = gt;
 313         ppgtt->vm.i915 = i915;
 314         ppgtt->vm.dma = i915->drm.dev;
 315         ppgtt->vm.total = BIT_ULL(RUNTIME_INFO(i915)->ppgtt_size);
 316         ppgtt->vm.lmem_pt_obj_flags = lmem_pt_obj_flags;
 317
 318         dma_resv_init(&ppgtt->vm._resv);
 319         i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
 320
 321         ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
 322         ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
 323 }