drm/i915/gtt: Use a common type for page directories

All page directories are identical in function, only the position in the
hierarchy differ. Use same base type for directory functionality.

v2: cleanup, size always 512, init to null

Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Matthew Auld <matthew.william.auld@gmail.com>
Cc: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>
Signed-off-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20190614164350.30415-2-mika.kuoppala@linux.intel.com

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 6200060..0f2c22a 100644 (file)
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -1027,7 +1027,7 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data)
 
        if (i915_vm_is_4lvl(vm)) {
                struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-               const dma_addr_t pd_daddr = px_dma(&ppgtt->pml4);
+               const dma_addr_t pd_daddr = px_dma(ppgtt->pd);
 
                cs = intel_ring_begin(rq, 6);
                if (IS_ERR(cs))

diff --git a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h
index 5ef932d..6bf3473 100644 (file)
--- a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h
+++ b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h
@@ -55,7 +55,7 @@
 
 #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \
        u32 *reg_state__ = (reg_state); \
-       const u64 addr__ = px_dma(&ppgtt->pml4); \
+       const u64 addr__ = px_dma(ppgtt->pd); \
        (reg_state__)[CTX_PDP0_UDW + 1] = upper_32_bits(addr__); \
        (reg_state__)[CTX_PDP0_LDW + 1] = lower_32_bits(addr__); \
 } while (0)

diff --git a/drivers/gpu/drm/i915/gt/intel_ringbuffer.c b/drivers/gpu/drm/i915/gt/intel_ringbuffer.c
index 019bf03..bd35ad2 100644 (file)
--- a/drivers/gpu/drm/i915/gt/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/gt/intel_ringbuffer.c
@@ -1484,7 +1484,7 @@ static int load_pd_dir(struct i915_request *rq, const struct i915_ppgtt *ppgtt)
 
        *cs++ = MI_LOAD_REGISTER_IMM(1);
        *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
-       *cs++ = ppgtt->pd.base.ggtt_offset << 10;
+       *cs++ = ppgtt->pd->base.ggtt_offset << 10;
 
        intel_ring_advance(rq, cs);
 

diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 8f0bd1e..5f4fa5d 100644 (file)
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -375,11 +375,13 @@ static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload,
                return -EINVAL;
 
        if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
-               px_dma(&ppgtt->pml4) = mm->ppgtt_mm.shadow_pdps[0];
+               px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0];
        } else {
                for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) {
-                       px_dma(ppgtt->pdp.page_directory[i]) =
-                               mm->ppgtt_mm.shadow_pdps[i];
+                       struct i915_page_directory * const pd =
+                               i915_pd_entry(ppgtt->pd, i);
+
+                       px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i];
                }
        }
 
@@ -1107,11 +1109,14 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s,
        int i;
 
        if (i915_vm_is_4lvl(&ppgtt->vm)) {
-               px_dma(&ppgtt->pml4) = s->i915_context_pml4;
+               px_dma(ppgtt->pd) = s->i915_context_pml4;
        } else {
-               for (i = 0; i < GEN8_3LVL_PDPES; i++)
-                       px_dma(ppgtt->pdp.page_directory[i]) =
-                               s->i915_context_pdps[i];
+               for (i = 0; i < GEN8_3LVL_PDPES; i++) {
+                       struct i915_page_directory * const pd =
+                               i915_pd_entry(ppgtt->pd, i);
+
+                       px_dma(pd) = s->i915_context_pdps[i];
+               }
        }
 }
 
@@ -1165,11 +1170,14 @@ i915_context_ppgtt_root_save(struct intel_vgpu_submission *s,
        int i;
 
        if (i915_vm_is_4lvl(&ppgtt->vm)) {
-               s->i915_context_pml4 = px_dma(&ppgtt->pml4);
+               s->i915_context_pml4 = px_dma(ppgtt->pd);
        } else {
-               for (i = 0; i < GEN8_3LVL_PDPES; i++)
-                       s->i915_context_pdps[i] =
-                               px_dma(ppgtt->pdp.page_directory[i]);
+               for (i = 0; i < GEN8_3LVL_PDPES; i++) {
+                       struct i915_page_directory * const pd =
+                               i915_pd_entry(ppgtt->pd, i);
+
+                       s->i915_context_pdps[i] = px_dma(pd);
+               }
        }
 }
 

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 0bad7bf..bd3dbd4 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -662,7 +662,8 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
                return ERR_PTR(-ENOMEM);
        }
 
-       atomic_set(&pt->used_ptes, 0);
+       atomic_set(&pt->used, 0);
+
        return pt;
 }
 
@@ -684,11 +685,30 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
        fill32_px(vm, pt, vm->scratch_pte);
 }
 
-static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
+static struct i915_page_directory *__alloc_pd(void)
 {
        struct i915_page_directory *pd;
 
        pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
+
+       if (unlikely(!pd))
+               return NULL;
+
+       memset(&pd->base, 0, sizeof(pd->base));
+       atomic_set(&pd->used, 0);
+       spin_lock_init(&pd->lock);
+
+       /* for safety */
+       pd->entry[0] = NULL;
+
+       return pd;
+}
+
+static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
+{
+       struct i915_page_directory *pd;
+
+       pd = __alloc_pd();
        if (unlikely(!pd))
                return ERR_PTR(-ENOMEM);
 
@@ -697,8 +717,6 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
                return ERR_PTR(-ENOMEM);
        }
 
-       atomic_set(&pd->used_pdes, 0);
-       spin_lock_init(&pd->lock);
        return pd;
 }
 
@@ -714,88 +732,56 @@ static void gen8_initialize_pd(struct i915_address_space *vm,
 {
        fill_px(vm, pd,
                gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
-       memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);
+       memset_p(pd->entry, vm->scratch_pt, I915_PDES);
 }
 
-static int __pdp_init(struct i915_address_space *vm,
-                     struct i915_page_directory_pointer *pdp)
+static struct i915_page_directory *alloc_pdp(struct i915_address_space *vm)
 {
-       const unsigned int pdpes = i915_pdpes_per_pdp(vm);
-
-       pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
-                                           I915_GFP_ALLOW_FAIL);
-       if (unlikely(!pdp->page_directory))
-               return -ENOMEM;
+       struct i915_page_directory *pdp;
 
-       memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
-
-       atomic_set(&pdp->used_pdpes, 0);
-       spin_lock_init(&pdp->lock);
-       return 0;
-}
-
-static void __pdp_fini(struct i915_page_directory_pointer *pdp)
-{
-       kfree(pdp->page_directory);
-       pdp->page_directory = NULL;
-}
-
-static struct i915_page_directory_pointer *
-alloc_pdp(struct i915_address_space *vm)
-{
-       struct i915_page_directory_pointer *pdp;
-       int ret = -ENOMEM;
-
-       GEM_BUG_ON(!i915_vm_is_4lvl(vm));
-
-       pdp = kmalloc(sizeof(*pdp), I915_GFP_ALLOW_FAIL);
+       pdp = __alloc_pd();
        if (!pdp)
                return ERR_PTR(-ENOMEM);
 
-       ret = __pdp_init(vm, pdp);
-       if (ret)
-               goto fail_bitmap;
-
-       ret = setup_px(vm, pdp);
-       if (ret)
-               goto fail_page_m;
+       if (i915_vm_is_4lvl(vm)) {
+               if (unlikely(setup_px(vm, pdp))) {
+                       kfree(pdp);
+                       return ERR_PTR(-ENOMEM);
+               }
+       }
 
        return pdp;
-
-fail_page_m:
-       __pdp_fini(pdp);
-fail_bitmap:
-       kfree(pdp);
-
-       return ERR_PTR(ret);
 }
 
 static void free_pdp(struct i915_address_space *vm,
-                    struct i915_page_directory_pointer *pdp)
+                    struct i915_page_directory *pdp)
 {
-       __pdp_fini(pdp);
-
-       if (!i915_vm_is_4lvl(vm))
-               return;
+       if (i915_vm_is_4lvl(vm))
+               cleanup_px(vm, pdp);
 
-       cleanup_px(vm, pdp);
        kfree(pdp);
 }
 
-static void gen8_initialize_pdp(struct i915_address_space *vm,
-                               struct i915_page_directory_pointer *pdp)
+static void gen8_initialize_4lvl_pdp(struct i915_address_space *vm,
+                                    struct i915_page_directory *pdp)
 {
        fill_px(vm, pdp,
                gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC));
+       memset_p(pdp->entry, vm->scratch_pd, 512);
+}
+
+static void gen8_initialize_3lvl_pdp(struct i915_address_space *vm,
+                                    struct i915_page_directory *pdp)
+{
+       memset_p(pdp->entry, vm->scratch_pd, GEN8_3LVL_PDPES);
 }
 
 static void gen8_initialize_pml4(struct i915_address_space *vm,
-                                struct i915_pml4 *pml4)
+                                struct i915_page_directory *pml4)
 {
        fill_px(vm, pml4,
                gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
-       memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
-       spin_lock_init(&pml4->lock);
+       memset_p(pml4->entry, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
 }
 
 /*
@@ -823,8 +809,8 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
        memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
        kunmap_atomic(vaddr);
 
-       GEM_BUG_ON(num_entries > atomic_read(&pt->used_ptes));
-       return !atomic_sub_return(num_entries, &pt->used_ptes);
+       GEM_BUG_ON(num_entries > atomic_read(&pt->used));
+       return !atomic_sub_return(num_entries, &pt->used);
 }
 
 static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
@@ -855,12 +841,12 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
                        continue;
 
                spin_lock(&pd->lock);
-               if (!atomic_read(&pt->used_ptes)) {
+               if (!atomic_read(&pt->used)) {
                        gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
-                       pd->page_table[pde] = vm->scratch_pt;
+                       pd->entry[pde] = vm->scratch_pt;
 
-                       GEM_BUG_ON(!atomic_read(&pd->used_pdes));
-                       atomic_dec(&pd->used_pdes);
+                       GEM_BUG_ON(!atomic_read(&pd->used));
+                       atomic_dec(&pd->used);
                        free = true;
                }
                spin_unlock(&pd->lock);
@@ -868,11 +854,11 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
                        free_pt(vm, pt);
        }
 
-       return !atomic_read(&pd->used_pdes);
+       return !atomic_read(&pd->used);
 }
 
 static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
-                               struct i915_page_directory_pointer *pdp,
+                               struct i915_page_directory *pdp,
                                struct i915_page_directory *pd,
                                unsigned int pdpe)
 {
@@ -890,7 +876,7 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
  * Caller can use the return value to update higher-level entries
  */
 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
-                                struct i915_page_directory_pointer *pdp,
+                                struct i915_page_directory * const pdp,
                                 u64 start, u64 length)
 {
        struct i915_page_directory *pd;
@@ -905,12 +891,12 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
                        continue;
 
                spin_lock(&pdp->lock);
-               if (!atomic_read(&pd->used_pdes)) {
+               if (!atomic_read(&pd->used)) {
                        gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
-                       pdp->page_directory[pdpe] = vm->scratch_pd;
+                       pdp->entry[pdpe] = vm->scratch_pd;
 
-                       GEM_BUG_ON(!atomic_read(&pdp->used_pdpes));
-                       atomic_dec(&pdp->used_pdpes);
+                       GEM_BUG_ON(!atomic_read(&pdp->used));
+                       atomic_dec(&pdp->used);
                        free = true;
                }
                spin_unlock(&pdp->lock);
@@ -918,17 +904,17 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
                        free_pd(vm, pd);
        }
 
-       return !atomic_read(&pdp->used_pdpes);
+       return !atomic_read(&pdp->used);
 }
 
 static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
                                  u64 start, u64 length)
 {
-       gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
+       gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length);
 }
 
-static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
-                                struct i915_page_directory_pointer *pdp,
+static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4,
+                                struct i915_page_directory *pdp,
                                 unsigned int pml4e)
 {
        gen8_ppgtt_pml4e_t *vaddr;
@@ -946,8 +932,8 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
                                  u64 start, u64 length)
 {
        struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-       struct i915_pml4 *pml4 = &ppgtt->pml4;
-       struct i915_page_directory_pointer *pdp;
+       struct i915_page_directory * const pml4 = ppgtt->pd;
+       struct i915_page_directory *pdp;
        unsigned int pml4e;
 
        GEM_BUG_ON(!i915_vm_is_4lvl(vm));
@@ -960,9 +946,9 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
                        continue;
 
                spin_lock(&pml4->lock);
-               if (!atomic_read(&pdp->used_pdpes)) {
+               if (!atomic_read(&pdp->used)) {
                        gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
-                       pml4->pdps[pml4e] = vm->scratch_pdp;
+                       pml4->entry[pml4e] = vm->scratch_pdp;
                        free = true;
                }
                spin_unlock(&pml4->lock);
@@ -999,7 +985,7 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
 
 static __always_inline bool
 gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
-                             struct i915_page_directory_pointer *pdp,
+                             struct i915_page_directory *pdp,
                              struct sgt_dma *iter,
                              struct gen8_insert_pte *idx,
                              enum i915_cache_level cache_level,
@@ -1011,8 +997,8 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
        bool ret;
 
        GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
-       pd = pdp->page_directory[idx->pdpe];
-       vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
+       pd = i915_pd_entry(pdp, idx->pdpe);
+       vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
        do {
                vaddr[idx->pte] = pte_encode | iter->dma;
 
@@ -1042,11 +1028,11 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
                                }
 
                                GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
-                               pd = pdp->page_directory[idx->pdpe];
+                               pd = pdp->entry[idx->pdpe];
                        }
 
                        kunmap_atomic(vaddr);
-                       vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
+                       vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
                }
        } while (1);
        kunmap_atomic(vaddr);
@@ -1063,14 +1049,14 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
        struct sgt_dma iter = sgt_dma(vma);
        struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
 
-       gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
+       gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,
                                      cache_level, flags);
 
        vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
 }
 
 static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
-                                          struct i915_page_directory_pointer **pdps,
+                                          struct i915_page_directory *pml4,
                                           struct sgt_dma *iter,
                                           enum i915_cache_level cache_level,
                                           u32 flags)
@@ -1081,8 +1067,9 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
 
        do {
                struct gen8_insert_pte idx = gen8_insert_pte(start);
-               struct i915_page_directory_pointer *pdp = pdps[idx.pml4e];
-               struct i915_page_directory *pd = pdp->page_directory[idx.pdpe];
+               struct i915_page_directory *pdp =
+                       i915_pdp_entry(pml4, idx.pml4e);
+               struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);
                unsigned int page_size;
                bool maybe_64K = false;
                gen8_pte_t encode = pte_encode;
@@ -1100,7 +1087,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
 
                        vaddr = kmap_atomic_px(pd);
                } else {
-                       struct i915_page_table *pt = pd->page_table[idx.pde];
+                       struct i915_page_table *pt = i915_pt_entry(pd, idx.pde);
 
                        index = idx.pte;
                        max = GEN8_PTES;
@@ -1175,7 +1162,8 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
                                u16 i;
 
                                encode = vma->vm->scratch_pte;
-                               vaddr = kmap_atomic_px(pd->page_table[idx.pde]);
+                               vaddr = kmap_atomic_px(i915_pt_entry(pd,
+                                                                    idx.pde));
 
                                for (i = 1; i < index; i += 16)
                                        memset64(vaddr + i, encode, 15);
@@ -1195,15 +1183,16 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 {
        struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
        struct sgt_dma iter = sgt_dma(vma);
-       struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
+       struct i915_page_directory * const pml4 = ppgtt->pd;
 
        if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
-               gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level,
+               gen8_ppgtt_insert_huge_entries(vma, pml4, &iter, cache_level,
                                               flags);
        } else {
                struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
 
-               while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],
+               while (gen8_ppgtt_insert_pte_entries(ppgtt,
+                                                    i915_pdp_entry(pml4, idx.pml4e++),
                                                     &iter, &idx, cache_level,
                                                     flags))
                        GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
@@ -1218,8 +1207,8 @@ static void gen8_free_page_tables(struct i915_address_space *vm,
        int i;
 
        for (i = 0; i < I915_PDES; i++) {
-               if (pd->page_table[i] != vm->scratch_pt)
-                       free_pt(vm, pd->page_table[i]);
+               if (pd->entry[i] != vm->scratch_pt)
+                       free_pt(vm, pd->entry[i]);
        }
 }
 
@@ -1278,7 +1267,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
        gen8_initialize_pt(vm, vm->scratch_pt);
        gen8_initialize_pd(vm, vm->scratch_pd);
        if (i915_vm_is_4lvl(vm))
-               gen8_initialize_pdp(vm, vm->scratch_pdp);
+               gen8_initialize_4lvl_pdp(vm, vm->scratch_pdp);
 
        return 0;
 
@@ -1300,7 +1289,7 @@ static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
        int i;
 
        if (i915_vm_is_4lvl(vm)) {
-               const u64 daddr = px_dma(&ppgtt->pml4);
+               const u64 daddr = px_dma(ppgtt->pd);
 
                I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
                I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
@@ -1337,17 +1326,17 @@ static void gen8_free_scratch(struct i915_address_space *vm)
 }
 
 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
-                                   struct i915_page_directory_pointer *pdp)
+                                   struct i915_page_directory *pdp)
 {
        const unsigned int pdpes = i915_pdpes_per_pdp(vm);
        int i;
 
        for (i = 0; i < pdpes; i++) {
-               if (pdp->page_directory[i] == vm->scratch_pd)
+               if (pdp->entry[i] == vm->scratch_pd)
                        continue;
 
-               gen8_free_page_tables(vm, pdp->page_directory[i]);
-               free_pd(vm, pdp->page_directory[i]);
+               gen8_free_page_tables(vm, pdp->entry[i]);
+               free_pd(vm, pdp->entry[i]);
        }
 
        free_pdp(vm, pdp);
@@ -1355,16 +1344,19 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
 
 static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
 {
+       struct i915_page_directory * const pml4 = ppgtt->pd;
        int i;
 
        for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
-               if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp)
+               struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
+
+               if (pdp == ppgtt->vm.scratch_pdp)
                        continue;
 
-               gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]);
+               gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
        }
 
-       cleanup_px(&ppgtt->vm, &ppgtt->pml4);
+       free_pd(&ppgtt->vm, pml4);
 }
 
 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
@@ -1378,7 +1370,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
        if (i915_vm_is_4lvl(vm))
                gen8_ppgtt_cleanup_4lvl(ppgtt);
        else
-               gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp);
+               gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);
 
        gen8_free_scratch(vm);
 }
@@ -1407,10 +1399,10 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
                        if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
                                gen8_initialize_pt(vm, pt);
 
-                       old = cmpxchg(&pd->page_table[pde], vm->scratch_pt, pt);
+                       old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);
                        if (old == vm->scratch_pt) {
                                gen8_ppgtt_set_pde(vm, pd, pt, pde);
-                               atomic_inc(&pd->used_pdes);
+                               atomic_inc(&pd->used);
                        } else {
                                free_pt(vm, pt);
                                pt = old;
@@ -1419,7 +1411,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
                        spin_lock(&pd->lock);
                }
 
-               atomic_add(count, &pt->used_ptes);
+               atomic_add(count, &pt->used);
        }
        spin_unlock(&pd->lock);
 
@@ -1431,7 +1423,7 @@ unwind:
 }
 
 static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
-                               struct i915_page_directory_pointer *pdp,
+                               struct i915_page_directory *pdp,
                                u64 start, u64 length)
 {
        struct i915_page_directory *pd;
@@ -1452,11 +1444,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 
                        gen8_initialize_pd(vm, pd);
 
-                       old = cmpxchg(&pdp->page_directory[pdpe],
-                                     vm->scratch_pd, pd);
+                       old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd);
                        if (old == vm->scratch_pd) {
                                gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
-                               atomic_inc(&pdp->used_pdpes);
+                               atomic_inc(&pdp->used);
                        } else {
                                free_pd(vm, pd);
                                pd = old;
@@ -1464,7 +1455,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 
                        spin_lock(&pdp->lock);
                }
-               atomic_inc(&pd->used_pdes);
+               atomic_inc(&pd->used);
                spin_unlock(&pdp->lock);
 
                ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
@@ -1472,7 +1463,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
                        goto unwind_pd;
 
                spin_lock(&pdp->lock);
-               atomic_dec(&pd->used_pdes);
+               atomic_dec(&pd->used);
        }
        spin_unlock(&pdp->lock);
 
@@ -1480,10 +1471,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 
 unwind_pd:
        spin_lock(&pdp->lock);
-       if (atomic_dec_and_test(&pd->used_pdes)) {
+       if (atomic_dec_and_test(&pd->used)) {
                gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
-               GEM_BUG_ON(!atomic_read(&pdp->used_pdpes));
-               atomic_dec(&pdp->used_pdpes);
+               GEM_BUG_ON(!atomic_read(&pdp->used));
+               atomic_dec(&pdp->used);
                free_pd(vm, pd);
        }
        spin_unlock(&pdp->lock);
@@ -1496,23 +1487,24 @@ static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
                                 u64 start, u64 length)
 {
        return gen8_ppgtt_alloc_pdp(vm,
-                                   &i915_vm_to_ppgtt(vm)->pdp, start, length);
+                                   i915_vm_to_ppgtt(vm)->pd, start, length);
 }
 
 static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
                                 u64 start, u64 length)
 {
        struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-       struct i915_pml4 *pml4 = &ppgtt->pml4;
-       struct i915_page_directory_pointer *pdp;
+       struct i915_page_directory * const pml4 = ppgtt->pd;
+       struct i915_page_directory *pdp;
        u64 from = start;
        u32 pml4e;
        int ret;
 
        spin_lock(&pml4->lock);
        gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
+
                if (pdp == vm->scratch_pdp) {
-                       struct i915_page_directory_pointer *old;
+                       struct i915_page_directory *old;
 
                        spin_unlock(&pml4->lock);
 
@@ -1520,9 +1512,9 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
                        if (IS_ERR(pdp))
                                goto unwind;
 
-                       gen8_initialize_pdp(vm, pdp);
+                       gen8_initialize_4lvl_pdp(vm, pdp);
 
-                       old = cmpxchg(&pml4->pdps[pml4e], vm->scratch_pdp, pdp);
+                       old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp);
                        if (old == vm->scratch_pdp) {
                                gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
                        } else {
@@ -1532,7 +1524,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
 
                        spin_lock(&pml4->lock);
                }
-               atomic_inc(&pdp->used_pdpes);
+               atomic_inc(&pdp->used);
                spin_unlock(&pml4->lock);
 
                ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
@@ -1540,7 +1532,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
                        goto unwind_pdp;
 
                spin_lock(&pml4->lock);
-               atomic_dec(&pdp->used_pdpes);
+               atomic_dec(&pdp->used);
        }
        spin_unlock(&pml4->lock);
 
@@ -1548,7 +1540,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
 
 unwind_pdp:
        spin_lock(&pml4->lock);
-       if (atomic_dec_and_test(&pdp->used_pdpes)) {
+       if (atomic_dec_and_test(&pdp->used)) {
                gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
                free_pdp(vm, pdp);
        }
@@ -1561,7 +1553,7 @@ unwind:
 static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
 {
        struct i915_address_space *vm = &ppgtt->vm;
-       struct i915_page_directory_pointer *pdp = &ppgtt->pdp;
+       struct i915_page_directory *pdp = ppgtt->pd;
        struct i915_page_directory *pd;
        u64 start = 0, length = ppgtt->vm.total;
        u64 from = start;
@@ -1574,10 +1566,12 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
 
                gen8_initialize_pd(vm, pd);
                gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
-               atomic_inc(&pdp->used_pdpes);
+
+               atomic_inc(&pdp->used);
        }
 
-       atomic_inc(&pdp->used_pdpes); /* never remove */
+       atomic_inc(&pdp->used); /* never remove */
+
        return 0;
 
 unwind:
@@ -1586,7 +1580,7 @@ unwind:
                gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
                free_pd(vm, pd);
        }
-       atomic_set(&pdp->used_pdpes, 0);
+       atomic_set(&pdp->used, 0);
        return -ENOMEM;
 }
 
@@ -1641,27 +1635,25 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
        if (err)
                goto err_free;
 
-       if (i915_vm_is_4lvl(&ppgtt->vm)) {
-               err = setup_px(&ppgtt->vm, &ppgtt->pml4);
-               if (err)
-                       goto err_scratch;
+       ppgtt->pd = alloc_pdp(&ppgtt->vm);
+       if (IS_ERR(ppgtt->pd)) {
+               err = PTR_ERR(ppgtt->pd);
+               goto err_scratch;
+       }
 
-               gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4);
+       if (i915_vm_is_4lvl(&ppgtt->vm)) {
+               gen8_initialize_pml4(&ppgtt->vm, ppgtt->pd);
 
                ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
                ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
                ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;
        } else {
-               err = __pdp_init(&ppgtt->vm, &ppgtt->pdp);
-               if (err)
-                       goto err_scratch;
+               gen8_initialize_3lvl_pdp(&ppgtt->vm, ppgtt->pd);
 
                if (intel_vgpu_active(i915)) {
                        err = gen8_preallocate_top_level_pdp(ppgtt);
-                       if (err) {
-                               __pdp_fini(&ppgtt->pdp);
-                               goto err_scratch;
-                       }
+                       if (err)
+                               goto err_pdp;
                }
 
                ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
@@ -1676,6 +1668,8 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 
        return ppgtt;
 
+err_pdp:
+       free_pdp(&ppgtt->vm, ppgtt->pd);
 err_scratch:
        gen8_free_scratch(&ppgtt->vm);
 err_free:
@@ -1741,15 +1735,16 @@ static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
                                   u64 start, u64 length)
 {
-       struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
-       unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
+       struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
+       const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
+       const gen6_pte_t scratch_pte = vm->scratch_pte;
        unsigned int pde = first_entry / GEN6_PTES;
        unsigned int pte = first_entry % GEN6_PTES;
        unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
-       const gen6_pte_t scratch_pte = vm->scratch_pte;
 
        while (num_entries) {
-               struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];
+               struct i915_page_table * const pt =
+                       i915_pt_entry(ppgtt->base.pd, pde++);
                const unsigned int count = min(num_entries, GEN6_PTES - pte);
                gen6_pte_t *vaddr;
 
@@ -1757,8 +1752,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 
                num_entries -= count;
 
-               GEM_BUG_ON(count > atomic_read(&pt->used_ptes));
-               if (!atomic_sub_return(count, &pt->used_ptes))
+               GEM_BUG_ON(count > atomic_read(&pt->used));
+               if (!atomic_sub_return(count, &pt->used))
                        ppgtt->scan_for_unused_pt = true;
 
                /*
@@ -1782,6 +1777,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
                                      u32 flags)
 {
        struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+       struct i915_page_directory * const pd = ppgtt->pd;
        unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE;
        unsigned act_pt = first_entry / GEN6_PTES;
        unsigned act_pte = first_entry % GEN6_PTES;
@@ -1789,9 +1785,9 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
        struct sgt_dma iter = sgt_dma(vma);
        gen6_pte_t *vaddr;
 
-       GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt);
+       GEM_BUG_ON(i915_pt_entry(pd, act_pt) == vm->scratch_pt);
 
-       vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
+       vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
        do {
                vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
 
@@ -1807,7 +1803,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 
                if (++act_pte == GEN6_PTES) {
                        kunmap_atomic(vaddr);
-                       vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);
+                       vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));
                        act_pte = 0;
                }
        } while (1);
@@ -1820,6 +1816,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
                               u64 start, u64 length)
 {
        struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
+       struct i915_page_directory * const pd = ppgtt->base.pd;
        struct i915_page_table *pt;
        intel_wakeref_t wakeref;
        u64 from = start;
@@ -1828,14 +1825,14 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 
        wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
 
-       spin_lock(&ppgtt->base.pd.lock);
-       gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {
+       spin_lock(&pd->lock);
+       gen6_for_each_pde(pt, pd, start, length, pde) {
                const unsigned int count = gen6_pte_count(start, length);
 
                if (pt == vm->scratch_pt) {
                        struct i915_page_table *old;
 
-                       spin_unlock(&ppgtt->base.pd.lock);
+                       spin_unlock(&pd->lock);
 
                        pt = alloc_pt(vm);
                        if (IS_ERR(pt))
@@ -1843,10 +1840,8 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 
                        gen6_initialize_pt(vm, pt);
 
-                       old = cmpxchg(&ppgtt->base.pd.page_table[pde],
-                                     vm->scratch_pt, pt);
+                       old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);
                        if (old == vm->scratch_pt) {
-                               ppgtt->base.pd.page_table[pde] = pt;
                                if (i915_vma_is_bound(ppgtt->vma,
                                                      I915_VMA_GLOBAL_BIND)) {
                                        gen6_write_pde(ppgtt, pde, pt);
@@ -1857,12 +1852,12 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
                                pt = old;
                        }
 
-                       spin_lock(&ppgtt->base.pd.lock);
+                       spin_lock(&pd->lock);
                }
 
-               atomic_add(count, &pt->used_ptes);
+               atomic_add(count, &pt->used);
        }
-       spin_unlock(&ppgtt->base.pd.lock);
+       spin_unlock(&pd->lock);
 
        if (flush) {
                mark_tlbs_dirty(&ppgtt->base);
@@ -1882,6 +1877,7 @@ unwind_out:
 static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
 {
        struct i915_address_space * const vm = &ppgtt->base.vm;
+       struct i915_page_directory * const pd = ppgtt->base.pd;
        struct i915_page_table *unused;
        u32 pde;
        int ret;
@@ -1901,9 +1897,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
        }
 
        gen6_initialize_pt(vm, vm->scratch_pt);
-       gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
-               ppgtt->base.pd.page_table[pde] = vm->scratch_pt;
-       spin_lock_init(&ppgtt->base.pd.lock);
+
+       gen6_for_all_pdes(unused, pd, pde)
+               pd->entry[pde] = vm->scratch_pt;
 
        return 0;
 }
@@ -1916,10 +1912,11 @@ static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
 
 static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
 {
+       struct i915_page_directory * const pd = ppgtt->base.pd;
        struct i915_page_table *pt;
        u32 pde;
 
-       gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)
+       gen6_for_all_pdes(pt, pd, pde)
                if (pt != ppgtt->base.vm.scratch_pt)
                        free_pt(&ppgtt->base.vm, pt);
 }
@@ -1983,6 +1980,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 
        gen6_ppgtt_free_pd(ppgtt);
        gen6_ppgtt_free_scratch(vm);
+       kfree(ppgtt->base.pd);
 }
 
 static int pd_vma_set_pages(struct i915_vma *vma)
@@ -2008,10 +2006,10 @@ static int pd_vma_bind(struct i915_vma *vma,
        struct i915_page_table *pt;
        unsigned int pde;
 
-       ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);
+       ppgtt->base.pd->base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);
        ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
 
-       gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)
+       gen6_for_all_pdes(pt, ppgtt->base.pd, pde)
                gen6_write_pde(ppgtt, pde, pt);
 
        mark_tlbs_dirty(&ppgtt->base);
@@ -2023,6 +2021,7 @@ static int pd_vma_bind(struct i915_vma *vma,
 static void pd_vma_unbind(struct i915_vma *vma)
 {
        struct gen6_ppgtt *ppgtt = vma->private;
+       struct i915_page_directory * const pd = ppgtt->base.pd;
        struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;
        struct i915_page_table *pt;
        unsigned int pde;
@@ -2031,12 +2030,12 @@ static void pd_vma_unbind(struct i915_vma *vma)
                return;
 
        /* Free all no longer used page tables */
-       gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {
-               if (atomic_read(&pt->used_ptes) || pt == scratch_pt)
+       gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
+               if (atomic_read(&pt->used) || pt == scratch_pt)
                        continue;
 
                free_pt(&ppgtt->base.vm, pt);
-               ppgtt->base.pd.page_table[pde] = scratch_pt;
+               pd->entry[pde] = scratch_pt;
        }
 
        ppgtt->scan_for_unused_pt = false;
@@ -2165,9 +2164,15 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
                goto err_free;
        }
 
+       ppgtt->base.pd = __alloc_pd();
+       if (!ppgtt->base.pd) {
+               err = -ENOMEM;
+               goto err_work;
+       }
+
        err = gen6_ppgtt_init_scratch(ppgtt);
        if (err)
-               goto err_work;
+               goto err_pd;
 
        ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
        if (IS_ERR(ppgtt->vma)) {
@@ -2179,6 +2184,8 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
 
 err_scratch:
        gen6_ppgtt_free_scratch(&ppgtt->base.vm);
+err_pd:
+       kfree(ppgtt->base.pd);
 err_work:
        kfree(ppgtt->work);
 err_free:

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 63fa357..812717c 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -248,28 +248,14 @@ struct i915_page_dma {
 
 struct i915_page_table {
        struct i915_page_dma base;
-       atomic_t used_ptes;
+       atomic_t used;
 };
 
 struct i915_page_directory {
        struct i915_page_dma base;
-
-       struct i915_page_table *page_table[I915_PDES]; /* PDEs */
-       atomic_t used_pdes;
-       spinlock_t lock;
-};
-
-struct i915_page_directory_pointer {
-       struct i915_page_dma base;
-       struct i915_page_directory **page_directory;
-       atomic_t used_pdpes;
-       spinlock_t lock;
-};
-
-struct i915_pml4 {
-       struct i915_page_dma base;
-       struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
+       atomic_t used;
        spinlock_t lock;
+       void *entry[512];
 };
 
 struct i915_vma_ops {
@@ -321,7 +307,7 @@ struct i915_address_space {
        struct i915_page_dma scratch_page;
        struct i915_page_table *scratch_pt;
        struct i915_page_directory *scratch_pd;
-       struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
+       struct i915_page_directory *scratch_pdp; /* GEN8+ & 48b PPGTT */
 
        /**
         * List of vma currently bound.
@@ -428,11 +414,7 @@ struct i915_ppgtt {
        struct i915_address_space vm;
 
        intel_engine_mask_t pd_dirty_engines;
-       union {
-               struct i915_pml4 pml4;          /* GEN8+ & 48b PPGTT */
-               struct i915_page_directory_pointer pdp; /* GEN8+ */
-               struct i915_page_directory pd;          /* GEN6-7 */
-       };
+       struct i915_page_directory *pd;
 };
 
 struct gen6_ppgtt {
@@ -466,7 +448,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base)
 #define gen6_for_each_pde(pt, pd, start, length, iter)                 \
        for (iter = gen6_pde_index(start);                              \
             length > 0 && iter < I915_PDES &&                          \
-               (pt = (pd)->page_table[iter], true);                    \
+                    (pt = i915_pt_entry(pd, iter), true);              \
             ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT);         \
                    temp = min(temp - start, length);                   \
                    start += temp, length -= temp; }), ++iter)
@@ -474,7 +456,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base)
 #define gen6_for_all_pdes(pt, pd, iter)                                        \
        for (iter = 0;                                                  \
             iter < I915_PDES &&                                        \
-               (pt = (pd)->page_table[iter], true);                    \
+                    (pt = i915_pt_entry(pd, iter), true);              \
             ++iter)
 
 static inline u32 i915_pte_index(u64 address, unsigned int pde_shift)
@@ -533,6 +515,27 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
        return GEN8_3LVL_PDPES;
 }
 
+static inline struct i915_page_table *
+i915_pt_entry(const struct i915_page_directory * const pd,
+             const unsigned short n)
+{
+       return pd->entry[n];
+}
+
+static inline struct i915_page_directory *
+i915_pd_entry(const struct i915_page_directory * const pdp,
+             const unsigned short n)
+{
+       return pdp->entry[n];
+}
+
+static inline struct i915_page_directory *
+i915_pdp_entry(const struct i915_page_directory * const pml4,
+              const unsigned short n)
+{
+       return pml4->entry[n];
+}
+
 /* Equivalent to the gen6 version, For each pde iterates over every pde
  * between from start until start + length. On gen8+ it simply iterates
  * over every page directory entry in a page directory.
@@ -540,7 +543,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
 #define gen8_for_each_pde(pt, pd, start, length, iter)                 \
        for (iter = gen8_pde_index(start);                              \
             length > 0 && iter < I915_PDES &&                          \
-               (pt = (pd)->page_table[iter], true);                    \
+                    (pt = i915_pt_entry(pd, iter), true);              \
             ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT);         \
                    temp = min(temp - start, length);                   \
                    start += temp, length -= temp; }), ++iter)
@@ -548,7 +551,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
 #define gen8_for_each_pdpe(pd, pdp, start, length, iter)               \
        for (iter = gen8_pdpe_index(start);                             \
             length > 0 && iter < i915_pdpes_per_pdp(vm) &&             \
-               (pd = (pdp)->page_directory[iter], true);               \
+                    (pd = i915_pd_entry(pdp, iter), true);             \
             ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT);        \
                    temp = min(temp - start, length);                   \
                    start += temp, length -= temp; }), ++iter)
@@ -556,7 +559,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
 #define gen8_for_each_pml4e(pdp, pml4, start, length, iter)            \
        for (iter = gen8_pml4e_index(start);                            \
             length > 0 && iter < GEN8_PML4ES_PER_PML4 &&               \
-               (pdp = (pml4)->pdps[iter], true);                       \
+                    (pdp = i915_pdp_entry(pml4, iter), true);          \
             ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT);    \
                    temp = min(temp - start, length);                   \
                    start += temp, length -= temp; }), ++iter)
@@ -589,7 +592,10 @@ static inline u64 gen8_pte_count(u64 address, u64 length)
 static inline dma_addr_t
 i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n)
 {
-       return px_dma(ppgtt->pdp.page_directory[n]);
+       struct i915_page_directory *pd;
+
+       pd = i915_pdp_entry(ppgtt->pd, n);
+       return px_dma(pd);
 }
 
 static inline struct i915_ggtt *