1 // SPDX-License-Identifier: MIT
3 * Copyright © 2021 Intel Corporation
7 #include "intel_display_types.h"
10 #include "gt/gen8_ppgtt.h"
13 struct i915_address_space vm;
15 struct drm_i915_gem_object *obj;
20 #define i915_is_dpt(vm) ((vm)->is_dpt)
22 static inline struct i915_dpt *
23 i915_vm_to_dpt(struct i915_address_space *vm)
25 BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
26 GEM_BUG_ON(!i915_is_dpt(vm));
27 return container_of(vm, struct i915_dpt, vm);
30 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
32 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
37 static void dpt_insert_page(struct i915_address_space *vm,
40 enum i915_cache_level level,
43 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
44 gen8_pte_t __iomem *base = dpt->iomem;
46 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
47 vm->pte_encode(addr, level, flags));
50 static void dpt_insert_entries(struct i915_address_space *vm,
52 enum i915_cache_level level,
55 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
56 gen8_pte_t __iomem *base = dpt->iomem;
57 const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags);
58 struct sgt_iter sgt_iter;
63 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
64 * not to allow the user to override access to a read only page.
67 i = vma->node.start / I915_GTT_PAGE_SIZE;
68 for_each_sgt_daddr(addr, sgt_iter, vma->pages)
69 gen8_set_pte(&base[i++], pte_encode | addr);
72 static void dpt_clear_range(struct i915_address_space *vm,
73 u64 start, u64 length)
77 static void dpt_bind_vma(struct i915_address_space *vm,
78 struct i915_vm_pt_stash *stash,
80 enum i915_cache_level cache_level,
83 struct drm_i915_gem_object *obj = vma->obj;
86 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
88 if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj))
89 pte_flags |= PTE_READ_ONLY;
90 if (i915_gem_object_is_lmem(obj))
93 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
95 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
98 * Without aliasing PPGTT there's no difference between
99 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
100 * upgrade to both bound if we bind either to avoid double-binding.
102 atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
105 static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma)
107 vm->clear_range(vm, vma->node.start, vma->size);
110 static void dpt_cleanup(struct i915_address_space *vm)
112 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
114 i915_gem_object_put(dpt->obj);
117 struct i915_vma *intel_dpt_pin(struct i915_address_space *vm)
119 struct drm_i915_private *i915 = vm->i915;
120 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
121 intel_wakeref_t wakeref;
122 struct i915_vma *vma;
124 struct i915_gem_ww_ctx ww;
127 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
128 atomic_inc(&i915->gpu_error.pending_fb_pin);
130 for_i915_gem_ww(&ww, err, true) {
131 err = i915_gem_object_lock(dpt->obj, &ww);
135 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 4096,
136 HAS_LMEM(i915) ? 0 : PIN_MAPPABLE);
142 iomem = i915_vma_pin_iomap(vma);
146 err = PTR_ERR(iomem);
156 atomic_dec(&i915->gpu_error.pending_fb_pin);
157 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
159 return err ? ERR_PTR(err) : vma;
162 void intel_dpt_unpin(struct i915_address_space *vm)
164 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
166 i915_vma_unpin_iomap(dpt->vma);
167 i915_vma_put(dpt->vma);
171 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
172 * @i915: device instance
174 * Restore the memory mapping during system resume for all framebuffers which
175 * are mapped to HW via a GGTT->DPT page table. The content of these page
176 * tables are not stored in the hibernation image during S4 and S3RST->S4
177 * transitions, so here we reprogram the PTE entries in those tables.
179 * This function must be called after the mappings in GGTT have been restored calling
180 * i915_ggtt_resume().
182 void intel_dpt_resume(struct drm_i915_private *i915)
184 struct drm_framebuffer *drm_fb;
186 if (!HAS_DISPLAY(i915))
189 mutex_lock(&i915->drm.mode_config.fb_lock);
190 drm_for_each_fb(drm_fb, &i915->drm) {
191 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
194 i915_ggtt_resume_vm(fb->dpt_vm);
196 mutex_unlock(&i915->drm.mode_config.fb_lock);
200 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
201 * @i915: device instance
203 * Suspend the memory mapping during system suspend for all framebuffers which
204 * are mapped to HW via a GGTT->DPT page table.
206 * This function must be called before the mappings in GGTT are suspended calling
207 * i915_ggtt_suspend().
209 void intel_dpt_suspend(struct drm_i915_private *i915)
211 struct drm_framebuffer *drm_fb;
213 if (!HAS_DISPLAY(i915))
216 mutex_lock(&i915->drm.mode_config.fb_lock);
218 drm_for_each_fb(drm_fb, &i915->drm) {
219 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
222 i915_ggtt_suspend_vm(fb->dpt_vm);
225 mutex_unlock(&i915->drm.mode_config.fb_lock);
228 struct i915_address_space *
229 intel_dpt_create(struct intel_framebuffer *fb)
231 struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;
232 struct drm_i915_private *i915 = to_i915(obj->dev);
233 struct drm_i915_gem_object *dpt_obj;
234 struct i915_address_space *vm;
235 struct i915_dpt *dpt;
239 if (intel_fb_needs_pot_stride_remap(fb))
240 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
242 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
244 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
247 dpt_obj = i915_gem_object_create_lmem(i915, size, 0);
249 dpt_obj = i915_gem_object_create_stolen(i915, size);
251 return ERR_CAST(dpt_obj);
253 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
255 i915_gem_object_put(dpt_obj);
259 dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
261 i915_gem_object_put(dpt_obj);
262 return ERR_PTR(-ENOMEM);
267 vm->gt = to_gt(i915);
269 vm->dma = i915->drm.dev;
270 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
273 i915_address_space_init(vm, VM_CLASS_DPT);
275 vm->insert_page = dpt_insert_page;
276 vm->clear_range = dpt_clear_range;
277 vm->insert_entries = dpt_insert_entries;
278 vm->cleanup = dpt_cleanup;
280 vm->vma_ops.bind_vma = dpt_bind_vma;
281 vm->vma_ops.unbind_vma = dpt_unbind_vma;
283 vm->pte_encode = gen8_ggtt_pte_encode;
290 void intel_dpt_destroy(struct i915_address_space *vm)
292 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
294 i915_vm_close(&dpt->vm);