2 * Copyright © 2010 Daniel Vetter
3 * Copyright © 2011-2014 Intel Corporation
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <linux/slab.h> /* fault-inject.h is not standalone! */
28 #include <linux/fault-inject.h>
29 #include <linux/log2.h>
30 #include <linux/random.h>
31 #include <linux/seq_file.h>
32 #include <linux/stop_machine.h>
34 #include <asm/set_memory.h>
37 #include <drm/i915_drm.h>
39 #include "display/intel_frontbuffer.h"
40 #include "gt/intel_gt.h"
41 #include "gt/intel_gt_requests.h"
44 #include "i915_scatterlist.h"
45 #include "i915_trace.h"
46 #include "i915_vgpu.h"
48 #define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
50 #if IS_ENABLED(CONFIG_DRM_I915_TRACE_GTT)
51 #define DBG(...) trace_printk(__VA_ARGS__)
57 * DOC: Global GTT views
59 * Background and previous state
61 * Historically objects could exists (be bound) in global GTT space only as
62 * singular instances with a view representing all of the object's backing pages
63 * in a linear fashion. This view will be called a normal view.
65 * To support multiple views of the same object, where the number of mapped
66 * pages is not equal to the backing store, or where the layout of the pages
67 * is not linear, concept of a GGTT view was added.
69 * One example of an alternative view is a stereo display driven by a single
70 * image. In this case we would have a framebuffer looking like this
76 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
77 * rendering. In contrast, fed to the display engine would be an alternative
78 * view which could look something like this:
83 * In this example both the size and layout of pages in the alternative view is
84 * different from the normal view.
86 * Implementation and usage
88 * GGTT views are implemented using VMAs and are distinguished via enum
89 * i915_ggtt_view_type and struct i915_ggtt_view.
91 * A new flavour of core GEM functions which work with GGTT bound objects were
92 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
93 * renaming in large amounts of code. They take the struct i915_ggtt_view
94 * parameter encapsulating all metadata required to implement a view.
96 * As a helper for callers which are only interested in the normal view,
97 * globally const i915_ggtt_view_normal singleton instance exists. All old core
98 * GEM API functions, the ones not taking the view parameter, are operating on,
99 * or with the normal GGTT view.
101 * Code wanting to add or use a new GGTT view needs to:
103 * 1. Add a new enum with a suitable name.
104 * 2. Extend the metadata in the i915_ggtt_view structure if required.
105 * 3. Add support to i915_get_vma_pages().
107 * New views are required to build a scatter-gather table from within the
108 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
109 * exists for the lifetime of an VMA.
111 * Core API is designed to have copy semantics which means that passed in
112 * struct i915_ggtt_view does not need to be persistent (left around after
113 * calling the core API functions).
117 #define as_pd(x) container_of((x), typeof(struct i915_page_directory), pt)
120 i915_get_ggtt_vma_pages(struct i915_vma *vma);
122 static void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
124 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
127 * Note that as an uncached mmio write, this will flush the
128 * WCB of the writes into the GGTT before it triggers the invalidate.
130 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
133 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
135 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
136 struct drm_i915_private *i915 = ggtt->vm.i915;
138 gen6_ggtt_invalidate(ggtt);
140 if (INTEL_GEN(i915) >= 12)
141 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,
142 GEN12_GUC_TLB_INV_CR_INVALIDATE);
144 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
147 static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt)
149 intel_gtt_chipset_flush();
152 static int ppgtt_bind_vma(struct i915_vma *vma,
153 enum i915_cache_level cache_level,
159 if (flags & I915_VMA_ALLOC) {
160 err = vma->vm->allocate_va_range(vma->vm,
161 vma->node.start, vma->size);
165 set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma));
168 /* Applicable to VLV, and gen8+ */
170 if (i915_gem_object_is_readonly(vma->obj))
171 pte_flags |= PTE_READ_ONLY;
173 GEM_BUG_ON(!test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)));
174 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
180 static void ppgtt_unbind_vma(struct i915_vma *vma)
182 if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)))
183 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
186 static int ppgtt_set_pages(struct i915_vma *vma)
188 GEM_BUG_ON(vma->pages);
190 vma->pages = vma->obj->mm.pages;
192 vma->page_sizes = vma->obj->mm.page_sizes;
197 static void clear_pages(struct i915_vma *vma)
199 GEM_BUG_ON(!vma->pages);
201 if (vma->pages != vma->obj->mm.pages) {
202 sg_free_table(vma->pages);
207 memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));
210 static u64 gen8_pte_encode(dma_addr_t addr,
211 enum i915_cache_level level,
214 gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW;
216 if (unlikely(flags & PTE_READ_ONLY))
220 case I915_CACHE_NONE:
221 pte |= PPAT_UNCACHED;
224 pte |= PPAT_DISPLAY_ELLC;
234 static u64 gen8_pde_encode(const dma_addr_t addr,
235 const enum i915_cache_level level)
237 u64 pde = _PAGE_PRESENT | _PAGE_RW;
239 if (level != I915_CACHE_NONE)
240 pde |= PPAT_CACHED_PDE;
242 pde |= PPAT_UNCACHED;
246 static u64 snb_pte_encode(dma_addr_t addr,
247 enum i915_cache_level level,
250 gen6_pte_t pte = GEN6_PTE_VALID;
251 pte |= GEN6_PTE_ADDR_ENCODE(addr);
254 case I915_CACHE_L3_LLC:
256 pte |= GEN6_PTE_CACHE_LLC;
258 case I915_CACHE_NONE:
259 pte |= GEN6_PTE_UNCACHED;
268 static u64 ivb_pte_encode(dma_addr_t addr,
269 enum i915_cache_level level,
272 gen6_pte_t pte = GEN6_PTE_VALID;
273 pte |= GEN6_PTE_ADDR_ENCODE(addr);
276 case I915_CACHE_L3_LLC:
277 pte |= GEN7_PTE_CACHE_L3_LLC;
280 pte |= GEN6_PTE_CACHE_LLC;
282 case I915_CACHE_NONE:
283 pte |= GEN6_PTE_UNCACHED;
292 static u64 byt_pte_encode(dma_addr_t addr,
293 enum i915_cache_level level,
296 gen6_pte_t pte = GEN6_PTE_VALID;
297 pte |= GEN6_PTE_ADDR_ENCODE(addr);
299 if (!(flags & PTE_READ_ONLY))
300 pte |= BYT_PTE_WRITEABLE;
302 if (level != I915_CACHE_NONE)
303 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
308 static u64 hsw_pte_encode(dma_addr_t addr,
309 enum i915_cache_level level,
312 gen6_pte_t pte = GEN6_PTE_VALID;
313 pte |= HSW_PTE_ADDR_ENCODE(addr);
315 if (level != I915_CACHE_NONE)
316 pte |= HSW_WB_LLC_AGE3;
321 static u64 iris_pte_encode(dma_addr_t addr,
322 enum i915_cache_level level,
325 gen6_pte_t pte = GEN6_PTE_VALID;
326 pte |= HSW_PTE_ADDR_ENCODE(addr);
329 case I915_CACHE_NONE:
332 pte |= HSW_WT_ELLC_LLC_AGE3;
335 pte |= HSW_WB_ELLC_LLC_AGE3;
342 static void stash_init(struct pagestash *stash)
344 pagevec_init(&stash->pvec);
345 spin_lock_init(&stash->lock);
348 static struct page *stash_pop_page(struct pagestash *stash)
350 struct page *page = NULL;
352 spin_lock(&stash->lock);
353 if (likely(stash->pvec.nr))
354 page = stash->pvec.pages[--stash->pvec.nr];
355 spin_unlock(&stash->lock);
360 static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec)
364 spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING);
366 nr = min_t(typeof(nr), pvec->nr, pagevec_space(&stash->pvec));
367 memcpy(stash->pvec.pages + stash->pvec.nr,
368 pvec->pages + pvec->nr - nr,
369 sizeof(pvec->pages[0]) * nr);
370 stash->pvec.nr += nr;
372 spin_unlock(&stash->lock);
377 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
379 struct pagevec stack;
382 if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
383 i915_gem_shrink_all(vm->i915);
385 page = stash_pop_page(&vm->free_pages);
390 return alloc_page(gfp);
392 /* Look in our global stash of WC pages... */
393 page = stash_pop_page(&vm->i915->mm.wc_stash);
398 * Otherwise batch allocate pages to amortize cost of set_pages_wc.
400 * We have to be careful as page allocation may trigger the shrinker
401 * (via direct reclaim) which will fill up the WC stash underneath us.
402 * So we add our WB pages into a temporary pvec on the stack and merge
403 * them into the WC stash after all the allocations are complete.
405 pagevec_init(&stack);
409 page = alloc_page(gfp);
413 stack.pages[stack.nr++] = page;
414 } while (pagevec_space(&stack));
416 if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) {
417 page = stack.pages[--stack.nr];
419 /* Merge spare WC pages to the global stash */
421 stash_push_pagevec(&vm->i915->mm.wc_stash, &stack);
423 /* Push any surplus WC pages onto the local VM stash */
425 stash_push_pagevec(&vm->free_pages, &stack);
428 /* Return unwanted leftovers */
429 if (unlikely(stack.nr)) {
430 WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr));
431 __pagevec_release(&stack);
437 static void vm_free_pages_release(struct i915_address_space *vm,
440 struct pagevec *pvec = &vm->free_pages.pvec;
441 struct pagevec stack;
443 lockdep_assert_held(&vm->free_pages.lock);
444 GEM_BUG_ON(!pagevec_count(pvec));
446 if (vm->pt_kmap_wc) {
448 * When we use WC, first fill up the global stash and then
449 * only if full immediately free the overflow.
451 stash_push_pagevec(&vm->i915->mm.wc_stash, pvec);
454 * As we have made some room in the VM's free_pages,
455 * we can wait for it to fill again. Unless we are
456 * inside i915_address_space_fini() and must
457 * immediately release the pages!
459 if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1))
463 * We have to drop the lock to allow ourselves to sleep,
464 * so take a copy of the pvec and clear the stash for
465 * others to use it as we sleep.
468 pagevec_reinit(pvec);
469 spin_unlock(&vm->free_pages.lock);
472 set_pages_array_wb(pvec->pages, pvec->nr);
474 spin_lock(&vm->free_pages.lock);
477 __pagevec_release(pvec);
480 static void vm_free_page(struct i915_address_space *vm, struct page *page)
483 * On !llc, we need to change the pages back to WB. We only do so
484 * in bulk, so we rarely need to change the page attributes here,
485 * but doing so requires a stop_machine() from deep inside arch/x86/mm.
486 * To make detection of the possible sleep more likely, use an
487 * unconditional might_sleep() for everybody.
490 spin_lock(&vm->free_pages.lock);
491 while (!pagevec_space(&vm->free_pages.pvec))
492 vm_free_pages_release(vm, false);
493 GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec) >= PAGEVEC_SIZE);
494 pagevec_add(&vm->free_pages.pvec, page);
495 spin_unlock(&vm->free_pages.lock);
498 static void i915_address_space_fini(struct i915_address_space *vm)
500 spin_lock(&vm->free_pages.lock);
501 if (pagevec_count(&vm->free_pages.pvec))
502 vm_free_pages_release(vm, true);
503 GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec));
504 spin_unlock(&vm->free_pages.lock);
506 drm_mm_takedown(&vm->mm);
508 mutex_destroy(&vm->mutex);
511 void __i915_vm_close(struct i915_address_space *vm)
513 struct i915_vma *vma, *vn;
515 mutex_lock(&vm->mutex);
516 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
517 struct drm_i915_gem_object *obj = vma->obj;
519 /* Keep the obj (and hence the vma) alive as _we_ destroy it */
520 if (!kref_get_unless_zero(&obj->base.refcount))
523 atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
524 WARN_ON(__i915_vma_unbind(vma));
525 i915_vma_destroy(vma);
527 i915_gem_object_put(obj);
529 GEM_BUG_ON(!list_empty(&vm->bound_list));
530 mutex_unlock(&vm->mutex);
533 static void __i915_vm_release(struct work_struct *work)
535 struct i915_address_space *vm =
536 container_of(work, struct i915_address_space, rcu.work);
539 i915_address_space_fini(vm);
544 void i915_vm_release(struct kref *kref)
546 struct i915_address_space *vm =
547 container_of(kref, struct i915_address_space, ref);
549 GEM_BUG_ON(i915_is_ggtt(vm));
550 trace_i915_ppgtt_release(vm);
552 queue_rcu_work(vm->i915->wq, &vm->rcu);
555 static void i915_address_space_init(struct i915_address_space *vm, int subclass)
558 INIT_RCU_WORK(&vm->rcu, __i915_vm_release);
559 atomic_set(&vm->open, 1);
562 * The vm->mutex must be reclaim safe (for use in the shrinker).
563 * Do a dummy acquire now under fs_reclaim so that any allocation
564 * attempt holding the lock is immediately reported by lockdep.
566 mutex_init(&vm->mutex);
567 lockdep_set_subclass(&vm->mutex, subclass);
568 i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);
570 GEM_BUG_ON(!vm->total);
571 drm_mm_init(&vm->mm, 0, vm->total);
572 vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
574 stash_init(&vm->free_pages);
576 INIT_LIST_HEAD(&vm->bound_list);
579 static int __setup_page_dma(struct i915_address_space *vm,
580 struct i915_page_dma *p,
583 p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL);
584 if (unlikely(!p->page))
587 p->daddr = dma_map_page_attrs(vm->dma,
588 p->page, 0, PAGE_SIZE,
589 PCI_DMA_BIDIRECTIONAL,
590 DMA_ATTR_SKIP_CPU_SYNC |
592 if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
593 vm_free_page(vm, p->page);
600 static int setup_page_dma(struct i915_address_space *vm,
601 struct i915_page_dma *p)
603 return __setup_page_dma(vm, p, __GFP_HIGHMEM);
606 static void cleanup_page_dma(struct i915_address_space *vm,
607 struct i915_page_dma *p)
609 dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
610 vm_free_page(vm, p->page);
613 #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
616 fill_page_dma(const struct i915_page_dma *p, const u64 val, unsigned int count)
618 kunmap_atomic(memset64(kmap_atomic(p->page), val, count));
621 #define fill_px(px, v) fill_page_dma(px_base(px), (v), PAGE_SIZE / sizeof(u64))
622 #define fill32_px(px, v) do { \
623 u64 v__ = lower_32_bits(v); \
624 fill_px((px), v__ << 32 | v__); \
628 setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
633 * In order to utilize 64K pages for an object with a size < 2M, we will
634 * need to support a 64K scratch page, given that every 16th entry for a
635 * page-table operating in 64K mode must point to a properly aligned 64K
636 * region, including any PTEs which happen to point to scratch.
638 * This is only relevant for the 48b PPGTT where we support
639 * huge-gtt-pages, see also i915_vma_insert(). However, as we share the
640 * scratch (read-only) between all vm, we create one 64k scratch page
643 size = I915_GTT_PAGE_SIZE_4K;
644 if (i915_vm_is_4lvl(vm) &&
645 HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
646 size = I915_GTT_PAGE_SIZE_64K;
649 gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL;
652 unsigned int order = get_order(size);
656 page = alloc_pages(gfp, order);
660 addr = dma_map_page_attrs(vm->dma,
662 PCI_DMA_BIDIRECTIONAL,
663 DMA_ATTR_SKIP_CPU_SYNC |
665 if (unlikely(dma_mapping_error(vm->dma, addr)))
668 if (unlikely(!IS_ALIGNED(addr, size)))
671 vm->scratch[0].base.page = page;
672 vm->scratch[0].base.daddr = addr;
673 vm->scratch_order = order;
677 dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL);
679 __free_pages(page, order);
681 if (size == I915_GTT_PAGE_SIZE_4K)
684 size = I915_GTT_PAGE_SIZE_4K;
685 gfp &= ~__GFP_NOWARN;
689 static void cleanup_scratch_page(struct i915_address_space *vm)
691 struct i915_page_dma *p = px_base(&vm->scratch[0]);
692 unsigned int order = vm->scratch_order;
694 dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT,
695 PCI_DMA_BIDIRECTIONAL);
696 __free_pages(p->page, order);
699 static void free_scratch(struct i915_address_space *vm)
703 if (!px_dma(&vm->scratch[0])) /* set to 0 on clones */
706 for (i = 1; i <= vm->top; i++) {
707 if (!px_dma(&vm->scratch[i]))
709 cleanup_page_dma(vm, px_base(&vm->scratch[i]));
712 cleanup_scratch_page(vm);
715 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
717 struct i915_page_table *pt;
719 pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL);
721 return ERR_PTR(-ENOMEM);
723 if (unlikely(setup_page_dma(vm, &pt->base))) {
725 return ERR_PTR(-ENOMEM);
728 atomic_set(&pt->used, 0);
732 static struct i915_page_directory *__alloc_pd(size_t sz)
734 struct i915_page_directory *pd;
736 pd = kzalloc(sz, I915_GFP_ALLOW_FAIL);
740 spin_lock_init(&pd->lock);
744 static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
746 struct i915_page_directory *pd;
748 pd = __alloc_pd(sizeof(*pd));
750 return ERR_PTR(-ENOMEM);
752 if (unlikely(setup_page_dma(vm, px_base(pd)))) {
754 return ERR_PTR(-ENOMEM);
760 static void free_pd(struct i915_address_space *vm, struct i915_page_dma *pd)
762 cleanup_page_dma(vm, pd);
766 #define free_px(vm, px) free_pd(vm, px_base(px))
769 write_dma_entry(struct i915_page_dma * const pdma,
770 const unsigned short idx,
771 const u64 encoded_entry)
773 u64 * const vaddr = kmap_atomic(pdma->page);
775 vaddr[idx] = encoded_entry;
776 kunmap_atomic(vaddr);
780 __set_pd_entry(struct i915_page_directory * const pd,
781 const unsigned short idx,
782 struct i915_page_dma * const to,
783 u64 (*encode)(const dma_addr_t, const enum i915_cache_level))
785 /* Each thread pre-pins the pd, and we may have a thread per pde. */
786 GEM_BUG_ON(atomic_read(px_used(pd)) > 2 * ARRAY_SIZE(pd->entry));
788 atomic_inc(px_used(pd));
790 write_dma_entry(px_base(pd), idx, encode(to->daddr, I915_CACHE_LLC));
793 #define set_pd_entry(pd, idx, to) \
794 __set_pd_entry((pd), (idx), px_base(to), gen8_pde_encode)
797 clear_pd_entry(struct i915_page_directory * const pd,
798 const unsigned short idx,
799 const struct i915_page_scratch * const scratch)
801 GEM_BUG_ON(atomic_read(px_used(pd)) == 0);
803 write_dma_entry(px_base(pd), idx, scratch->encode);
804 pd->entry[idx] = NULL;
805 atomic_dec(px_used(pd));
809 release_pd_entry(struct i915_page_directory * const pd,
810 const unsigned short idx,
811 struct i915_page_table * const pt,
812 const struct i915_page_scratch * const scratch)
816 if (atomic_add_unless(&pt->used, -1, 1))
819 spin_lock(&pd->lock);
820 if (atomic_dec_and_test(&pt->used)) {
821 clear_pd_entry(pd, idx, scratch);
824 spin_unlock(&pd->lock);
829 static void gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
831 struct drm_i915_private *dev_priv = ppgtt->vm.i915;
832 enum vgt_g2v_type msg;
836 atomic_inc(px_used(ppgtt->pd)); /* never remove */
838 atomic_dec(px_used(ppgtt->pd));
840 mutex_lock(&dev_priv->vgpu.lock);
842 if (i915_vm_is_4lvl(&ppgtt->vm)) {
843 const u64 daddr = px_dma(ppgtt->pd);
845 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
846 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
848 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
849 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
851 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
852 const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
854 I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
855 I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
858 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
859 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
862 /* g2v_notify atomically (via hv trap) consumes the message packet. */
863 I915_WRITE(vgtif_reg(g2v_notify), msg);
865 mutex_unlock(&dev_priv->vgpu.lock);
868 /* Index shifts into the pagetable are offset by GEN8_PTE_SHIFT [12] */
869 #define GEN8_PAGE_SIZE (SZ_4K) /* page and page-directory sizes are the same */
870 #define GEN8_PTE_SHIFT (ilog2(GEN8_PAGE_SIZE))
871 #define GEN8_PDES (GEN8_PAGE_SIZE / sizeof(u64))
872 #define gen8_pd_shift(lvl) ((lvl) * ilog2(GEN8_PDES))
873 #define gen8_pd_index(i, lvl) i915_pde_index((i), gen8_pd_shift(lvl))
874 #define __gen8_pte_shift(lvl) (GEN8_PTE_SHIFT + gen8_pd_shift(lvl))
875 #define __gen8_pte_index(a, lvl) i915_pde_index((a), __gen8_pte_shift(lvl))
877 static inline unsigned int
878 gen8_pd_range(u64 start, u64 end, int lvl, unsigned int *idx)
880 const int shift = gen8_pd_shift(lvl);
881 const u64 mask = ~0ull << gen8_pd_shift(lvl + 1);
883 GEM_BUG_ON(start >= end);
884 end += ~mask >> gen8_pd_shift(1);
886 *idx = i915_pde_index(start, shift);
887 if ((start ^ end) & mask)
888 return GEN8_PDES - *idx;
890 return i915_pde_index(end, shift) - *idx;
893 static inline bool gen8_pd_contains(u64 start, u64 end, int lvl)
895 const u64 mask = ~0ull << gen8_pd_shift(lvl + 1);
897 GEM_BUG_ON(start >= end);
898 return (start ^ end) & mask && (start & ~mask) == 0;
901 static inline unsigned int gen8_pt_count(u64 start, u64 end)
903 GEM_BUG_ON(start >= end);
904 if ((start ^ end) >> gen8_pd_shift(1))
905 return GEN8_PDES - (start & (GEN8_PDES - 1));
910 static inline unsigned int gen8_pd_top_count(const struct i915_address_space *vm)
912 unsigned int shift = __gen8_pte_shift(vm->top);
913 return (vm->total + (1ull << shift) - 1) >> shift;
916 static inline struct i915_page_directory *
917 gen8_pdp_for_page_index(struct i915_address_space * const vm, const u64 idx)
919 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);
924 return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top));
927 static inline struct i915_page_directory *
928 gen8_pdp_for_page_address(struct i915_address_space * const vm, const u64 addr)
930 return gen8_pdp_for_page_index(vm, addr >> GEN8_PTE_SHIFT);
933 static void __gen8_ppgtt_cleanup(struct i915_address_space *vm,
934 struct i915_page_directory *pd,
938 void **pde = pd->entry;
944 __gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1);
945 } while (pde++, --count);
951 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
953 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
955 if (intel_vgpu_active(vm->i915))
956 gen8_ppgtt_notify_vgt(ppgtt, false);
958 __gen8_ppgtt_cleanup(vm, ppgtt->pd, gen8_pd_top_count(vm), vm->top);
962 static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm,
963 struct i915_page_directory * const pd,
964 u64 start, const u64 end, int lvl)
966 const struct i915_page_scratch * const scratch = &vm->scratch[lvl];
967 unsigned int idx, len;
969 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT);
971 len = gen8_pd_range(start, end, lvl--, &idx);
972 DBG("%s(%p):{ lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d }\n",
973 __func__, vm, lvl + 1, start, end,
974 idx, len, atomic_read(px_used(pd)));
975 GEM_BUG_ON(!len || len >= atomic_read(px_used(pd)));
978 struct i915_page_table *pt = pd->entry[idx];
980 if (atomic_fetch_inc(&pt->used) >> gen8_pd_shift(1) &&
981 gen8_pd_contains(start, end, lvl)) {
982 DBG("%s(%p):{ lvl:%d, idx:%d, start:%llx, end:%llx } removing pd\n",
983 __func__, vm, lvl + 1, idx, start, end);
984 clear_pd_entry(pd, idx, scratch);
985 __gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl);
986 start += (u64)I915_PDES << gen8_pd_shift(lvl);
991 start = __gen8_ppgtt_clear(vm, as_pd(pt),
997 count = gen8_pt_count(start, end);
998 DBG("%s(%p):{ lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d } removing pte\n",
999 __func__, vm, lvl, start, end,
1000 gen8_pd_index(start, 0), count,
1001 atomic_read(&pt->used));
1002 GEM_BUG_ON(!count || count >= atomic_read(&pt->used));
1004 vaddr = kmap_atomic_px(pt);
1005 memset64(vaddr + gen8_pd_index(start, 0),
1006 vm->scratch[0].encode,
1008 kunmap_atomic(vaddr);
1010 atomic_sub(count, &pt->used);
1014 if (release_pd_entry(pd, idx, pt, scratch))
1016 } while (idx++, --len);
1021 static void gen8_ppgtt_clear(struct i915_address_space *vm,
1022 u64 start, u64 length)
1024 GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));
1025 GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));
1026 GEM_BUG_ON(range_overflows(start, length, vm->total));
1028 start >>= GEN8_PTE_SHIFT;
1029 length >>= GEN8_PTE_SHIFT;
1030 GEM_BUG_ON(length == 0);
1032 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,
1033 start, start + length, vm->top);
1036 static int __gen8_ppgtt_alloc(struct i915_address_space * const vm,
1037 struct i915_page_directory * const pd,
1038 u64 * const start, const u64 end, int lvl)
1040 const struct i915_page_scratch * const scratch = &vm->scratch[lvl];
1041 struct i915_page_table *alloc = NULL;
1042 unsigned int idx, len;
1045 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT);
1047 len = gen8_pd_range(*start, end, lvl--, &idx);
1048 DBG("%s(%p):{ lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d }\n",
1049 __func__, vm, lvl + 1, *start, end,
1050 idx, len, atomic_read(px_used(pd)));
1051 GEM_BUG_ON(!len || (idx + len - 1) >> gen8_pd_shift(1));
1053 spin_lock(&pd->lock);
1054 GEM_BUG_ON(!atomic_read(px_used(pd))); /* Must be pinned! */
1056 struct i915_page_table *pt = pd->entry[idx];
1059 spin_unlock(&pd->lock);
1061 DBG("%s(%p):{ lvl:%d, idx:%d } allocating new tree\n",
1062 __func__, vm, lvl + 1, idx);
1064 pt = fetch_and_zero(&alloc);
1067 pt = &alloc_pd(vm)->pt;
1074 fill_px(pt, vm->scratch[lvl].encode);
1084 if (intel_vgpu_active(vm->i915) ||
1085 gen8_pt_count(*start, end) < I915_PDES)
1086 fill_px(pt, vm->scratch[lvl].encode);
1089 spin_lock(&pd->lock);
1090 if (likely(!pd->entry[idx]))
1091 set_pd_entry(pd, idx, pt);
1093 alloc = pt, pt = pd->entry[idx];
1097 atomic_inc(&pt->used);
1098 spin_unlock(&pd->lock);
1100 ret = __gen8_ppgtt_alloc(vm, as_pd(pt),
1102 if (unlikely(ret)) {
1103 if (release_pd_entry(pd, idx, pt, scratch))
1108 spin_lock(&pd->lock);
1109 atomic_dec(&pt->used);
1110 GEM_BUG_ON(!atomic_read(&pt->used));
1112 unsigned int count = gen8_pt_count(*start, end);
1114 DBG("%s(%p):{ lvl:%d, start:%llx, end:%llx, idx:%d, len:%d, used:%d } inserting pte\n",
1115 __func__, vm, lvl, *start, end,
1116 gen8_pd_index(*start, 0), count,
1117 atomic_read(&pt->used));
1119 atomic_add(count, &pt->used);
1120 /* All other pdes may be simultaneously removed */
1121 GEM_BUG_ON(atomic_read(&pt->used) > 2 * I915_PDES);
1124 } while (idx++, --len);
1125 spin_unlock(&pd->lock);
1132 static int gen8_ppgtt_alloc(struct i915_address_space *vm,
1133 u64 start, u64 length)
1138 GEM_BUG_ON(!IS_ALIGNED(start, BIT_ULL(GEN8_PTE_SHIFT)));
1139 GEM_BUG_ON(!IS_ALIGNED(length, BIT_ULL(GEN8_PTE_SHIFT)));
1140 GEM_BUG_ON(range_overflows(start, length, vm->total));
1142 start >>= GEN8_PTE_SHIFT;
1143 length >>= GEN8_PTE_SHIFT;
1144 GEM_BUG_ON(length == 0);
1147 err = __gen8_ppgtt_alloc(vm, i915_vm_to_ppgtt(vm)->pd,
1148 &start, start + length, vm->top);
1149 if (unlikely(err && from != start))
1150 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,
1151 from, start, vm->top);
1156 static inline struct sgt_dma {
1157 struct scatterlist *sg;
1158 dma_addr_t dma, max;
1159 } sgt_dma(struct i915_vma *vma) {
1160 struct scatterlist *sg = vma->pages->sgl;
1161 dma_addr_t addr = sg_dma_address(sg);
1162 return (struct sgt_dma) { sg, addr, addr + sg->length };
1165 static __always_inline u64
1166 gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt,
1167 struct i915_page_directory *pdp,
1168 struct sgt_dma *iter,
1170 enum i915_cache_level cache_level,
1173 struct i915_page_directory *pd;
1174 const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
1177 pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2));
1178 vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1)));
1180 vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma;
1182 iter->dma += I915_GTT_PAGE_SIZE;
1183 if (iter->dma >= iter->max) {
1184 iter->sg = __sg_next(iter->sg);
1190 iter->dma = sg_dma_address(iter->sg);
1191 iter->max = iter->dma + iter->sg->length;
1194 if (gen8_pd_index(++idx, 0) == 0) {
1195 if (gen8_pd_index(idx, 1) == 0) {
1196 /* Limited by sg length for 3lvl */
1197 if (gen8_pd_index(idx, 2) == 0)
1200 pd = pdp->entry[gen8_pd_index(idx, 2)];
1203 kunmap_atomic(vaddr);
1204 vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1)));
1207 kunmap_atomic(vaddr);
1212 static void gen8_ppgtt_insert_huge(struct i915_vma *vma,
1213 struct sgt_dma *iter,
1214 enum i915_cache_level cache_level,
1217 const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
1218 u64 start = vma->node.start;
1219 dma_addr_t rem = iter->sg->length;
1221 GEM_BUG_ON(!i915_vm_is_4lvl(vma->vm));
1224 struct i915_page_directory * const pdp =
1225 gen8_pdp_for_page_address(vma->vm, start);
1226 struct i915_page_directory * const pd =
1227 i915_pd_entry(pdp, __gen8_pte_index(start, 2));
1228 gen8_pte_t encode = pte_encode;
1229 unsigned int maybe_64K = -1;
1230 unsigned int page_size;
1234 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&
1235 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
1236 rem >= I915_GTT_PAGE_SIZE_2M &&
1237 !__gen8_pte_index(start, 0)) {
1238 index = __gen8_pte_index(start, 1);
1239 encode |= GEN8_PDE_PS_2M;
1240 page_size = I915_GTT_PAGE_SIZE_2M;
1242 vaddr = kmap_atomic_px(pd);
1244 struct i915_page_table *pt =
1245 i915_pt_entry(pd, __gen8_pte_index(start, 1));
1247 index = __gen8_pte_index(start, 0);
1248 page_size = I915_GTT_PAGE_SIZE;
1251 vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
1252 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1253 (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1254 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE))
1255 maybe_64K = __gen8_pte_index(start, 1);
1257 vaddr = kmap_atomic_px(pt);
1261 GEM_BUG_ON(iter->sg->length < page_size);
1262 vaddr[index++] = encode | iter->dma;
1265 iter->dma += page_size;
1267 if (iter->dma >= iter->max) {
1268 iter->sg = __sg_next(iter->sg);
1272 rem = iter->sg->length;
1273 iter->dma = sg_dma_address(iter->sg);
1274 iter->max = iter->dma + rem;
1276 if (maybe_64K != -1 && index < I915_PDES &&
1277 !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1278 (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1279 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)))
1282 if (unlikely(!IS_ALIGNED(iter->dma, page_size)))
1285 } while (rem >= page_size && index < I915_PDES);
1287 kunmap_atomic(vaddr);
1290 * Is it safe to mark the 2M block as 64K? -- Either we have
1291 * filled whole page-table with 64K entries, or filled part of
1292 * it and have reached the end of the sg table and we have
1295 if (maybe_64K != -1 &&
1296 (index == I915_PDES ||
1297 (i915_vm_has_scratch_64K(vma->vm) &&
1298 !iter->sg && IS_ALIGNED(vma->node.start +
1300 I915_GTT_PAGE_SIZE_2M)))) {
1301 vaddr = kmap_atomic_px(pd);
1302 vaddr[maybe_64K] |= GEN8_PDE_IPS_64K;
1303 kunmap_atomic(vaddr);
1304 page_size = I915_GTT_PAGE_SIZE_64K;
1307 * We write all 4K page entries, even when using 64K
1308 * pages. In order to verify that the HW isn't cheating
1309 * by using the 4K PTE instead of the 64K PTE, we want
1310 * to remove all the surplus entries. If the HW skipped
1311 * the 64K PTE, it will read/write into the scratch page
1312 * instead - which we detect as missing results during
1315 if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {
1318 encode = vma->vm->scratch[0].encode;
1319 vaddr = kmap_atomic_px(i915_pt_entry(pd, maybe_64K));
1321 for (i = 1; i < index; i += 16)
1322 memset64(vaddr + i, encode, 15);
1324 kunmap_atomic(vaddr);
1328 vma->page_sizes.gtt |= page_size;
1332 static void gen8_ppgtt_insert(struct i915_address_space *vm,
1333 struct i915_vma *vma,
1334 enum i915_cache_level cache_level,
1337 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);
1338 struct sgt_dma iter = sgt_dma(vma);
1340 if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
1341 gen8_ppgtt_insert_huge(vma, &iter, cache_level, flags);
1343 u64 idx = vma->node.start >> GEN8_PTE_SHIFT;
1346 struct i915_page_directory * const pdp =
1347 gen8_pdp_for_page_index(vm, idx);
1349 idx = gen8_ppgtt_insert_pte(ppgtt, pdp, &iter, idx,
1350 cache_level, flags);
1353 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1357 static int gen8_init_scratch(struct i915_address_space *vm)
1363 * If everybody agrees to not to write into the scratch page,
1364 * we can reuse it for all vm, keeping contexts and processes separate.
1366 if (vm->has_read_only &&
1367 vm->i915->kernel_context &&
1368 vm->i915->kernel_context->vm) {
1369 struct i915_address_space *clone =
1370 rcu_dereference_protected(vm->i915->kernel_context->vm,
1373 GEM_BUG_ON(!clone->has_read_only);
1375 vm->scratch_order = clone->scratch_order;
1376 memcpy(vm->scratch, clone->scratch, sizeof(vm->scratch));
1377 px_dma(&vm->scratch[0]) = 0; /* no xfer of ownership */
1381 ret = setup_scratch_page(vm, __GFP_HIGHMEM);
1385 vm->scratch[0].encode =
1386 gen8_pte_encode(px_dma(&vm->scratch[0]),
1387 I915_CACHE_LLC, vm->has_read_only);
1389 for (i = 1; i <= vm->top; i++) {
1390 if (unlikely(setup_page_dma(vm, px_base(&vm->scratch[i]))))
1393 fill_px(&vm->scratch[i], vm->scratch[i - 1].encode);
1394 vm->scratch[i].encode =
1395 gen8_pde_encode(px_dma(&vm->scratch[i]),
1406 static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
1408 struct i915_address_space *vm = &ppgtt->vm;
1409 struct i915_page_directory *pd = ppgtt->pd;
1412 GEM_BUG_ON(vm->top != 2);
1413 GEM_BUG_ON(gen8_pd_top_count(vm) != GEN8_3LVL_PDPES);
1415 for (idx = 0; idx < GEN8_3LVL_PDPES; idx++) {
1416 struct i915_page_directory *pde;
1420 return PTR_ERR(pde);
1422 fill_px(pde, vm->scratch[1].encode);
1423 set_pd_entry(pd, idx, pde);
1424 atomic_inc(px_used(pde)); /* keep pinned */
1431 static void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt)
1433 struct drm_i915_private *i915 = gt->i915;
1436 ppgtt->vm.i915 = i915;
1437 ppgtt->vm.dma = &i915->drm.pdev->dev;
1438 ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);
1440 i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
1442 ppgtt->vm.vma_ops.bind_vma = ppgtt_bind_vma;
1443 ppgtt->vm.vma_ops.unbind_vma = ppgtt_unbind_vma;
1444 ppgtt->vm.vma_ops.set_pages = ppgtt_set_pages;
1445 ppgtt->vm.vma_ops.clear_pages = clear_pages;
1448 static struct i915_page_directory *
1449 gen8_alloc_top_pd(struct i915_address_space *vm)
1451 const unsigned int count = gen8_pd_top_count(vm);
1452 struct i915_page_directory *pd;
1454 GEM_BUG_ON(count > ARRAY_SIZE(pd->entry));
1456 pd = __alloc_pd(offsetof(typeof(*pd), entry[count]));
1458 return ERR_PTR(-ENOMEM);
1460 if (unlikely(setup_page_dma(vm, px_base(pd)))) {
1462 return ERR_PTR(-ENOMEM);
1465 fill_page_dma(px_base(pd), vm->scratch[vm->top].encode, count);
1466 atomic_inc(px_used(pd)); /* mark as pinned */
1471 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1472 * with a net effect resembling a 2-level page table in normal x86 terms. Each
1473 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1477 static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
1479 struct i915_ppgtt *ppgtt;
1482 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1484 return ERR_PTR(-ENOMEM);
1486 ppgtt_init(ppgtt, &i915->gt);
1487 ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;
1490 * From bdw, there is hw support for read-only pages in the PPGTT.
1492 * Gen11 has HSDES#:1807136187 unresolved. Disable ro support
1495 * Gen12 has inherited the same read-only fault issue from gen11.
1497 ppgtt->vm.has_read_only = !IS_GEN_RANGE(i915, 11, 12);
1499 /* There are only few exceptions for gen >=6. chv and bxt.
1500 * And we are not sure about the latter so play safe for now.
1502 if (IS_CHERRYVIEW(i915) || IS_BROXTON(i915))
1503 ppgtt->vm.pt_kmap_wc = true;
1505 err = gen8_init_scratch(&ppgtt->vm);
1509 ppgtt->pd = gen8_alloc_top_pd(&ppgtt->vm);
1510 if (IS_ERR(ppgtt->pd)) {
1511 err = PTR_ERR(ppgtt->pd);
1512 goto err_free_scratch;
1515 if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1516 err = gen8_preallocate_top_level_pdp(ppgtt);
1521 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND;
1522 ppgtt->vm.insert_entries = gen8_ppgtt_insert;
1523 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
1524 ppgtt->vm.clear_range = gen8_ppgtt_clear;
1526 if (intel_vgpu_active(i915))
1527 gen8_ppgtt_notify_vgt(ppgtt, true);
1529 ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
1534 __gen8_ppgtt_cleanup(&ppgtt->vm, ppgtt->pd,
1535 gen8_pd_top_count(&ppgtt->vm), ppgtt->vm.top);
1537 free_scratch(&ppgtt->vm);
1540 return ERR_PTR(err);
1543 /* Write pde (index) from the page directory @pd to the page table @pt */
1544 static inline void gen6_write_pde(const struct gen6_ppgtt *ppgtt,
1545 const unsigned int pde,
1546 const struct i915_page_table *pt)
1548 /* Caller needs to make sure the write completes if necessary */
1549 iowrite32(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID,
1550 ppgtt->pd_addr + pde);
1553 static void gen7_ppgtt_enable(struct intel_gt *gt)
1555 struct drm_i915_private *i915 = gt->i915;
1556 struct intel_uncore *uncore = gt->uncore;
1557 struct intel_engine_cs *engine;
1558 enum intel_engine_id id;
1561 intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B);
1563 ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
1564 if (IS_HASWELL(i915)) {
1565 ecochk |= ECOCHK_PPGTT_WB_HSW;
1567 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1568 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1570 intel_uncore_write(uncore, GAM_ECOCHK, ecochk);
1572 for_each_engine(engine, gt, id) {
1573 /* GFX_MODE is per-ring on gen7+ */
1574 ENGINE_WRITE(engine,
1576 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1580 static void gen6_ppgtt_enable(struct intel_gt *gt)
1582 struct intel_uncore *uncore = gt->uncore;
1584 intel_uncore_rmw(uncore,
1587 ECOBITS_SNB_BIT | ECOBITS_PPGTT_CACHE64B);
1589 intel_uncore_rmw(uncore,
1592 GAB_CTL_CONT_AFTER_PAGEFAULT);
1594 intel_uncore_rmw(uncore,
1597 ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1599 if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */
1600 intel_uncore_write(uncore,
1602 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1605 /* PPGTT support for Sandybdrige/Gen6 and later */
1606 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1607 u64 start, u64 length)
1609 struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1610 const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
1611 const gen6_pte_t scratch_pte = vm->scratch[0].encode;
1612 unsigned int pde = first_entry / GEN6_PTES;
1613 unsigned int pte = first_entry % GEN6_PTES;
1614 unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
1616 while (num_entries) {
1617 struct i915_page_table * const pt =
1618 i915_pt_entry(ppgtt->base.pd, pde++);
1619 const unsigned int count = min(num_entries, GEN6_PTES - pte);
1622 GEM_BUG_ON(px_base(pt) == px_base(&vm->scratch[1]));
1624 num_entries -= count;
1626 GEM_BUG_ON(count > atomic_read(&pt->used));
1627 if (!atomic_sub_return(count, &pt->used))
1628 ppgtt->scan_for_unused_pt = true;
1631 * Note that the hw doesn't support removing PDE on the fly
1632 * (they are cached inside the context with no means to
1633 * invalidate the cache), so we can only reset the PTE
1634 * entries back to scratch.
1637 vaddr = kmap_atomic_px(pt);
1638 memset32(vaddr + pte, scratch_pte, count);
1639 kunmap_atomic(vaddr);
1645 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1646 struct i915_vma *vma,
1647 enum i915_cache_level cache_level,
1650 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1651 struct i915_page_directory * const pd = ppgtt->pd;
1652 unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE;
1653 unsigned act_pt = first_entry / GEN6_PTES;
1654 unsigned act_pte = first_entry % GEN6_PTES;
1655 const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1656 struct sgt_dma iter = sgt_dma(vma);
1659 GEM_BUG_ON(pd->entry[act_pt] == &vm->scratch[1]);
1661 vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
1663 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1665 iter.dma += I915_GTT_PAGE_SIZE;
1666 if (iter.dma == iter.max) {
1667 iter.sg = __sg_next(iter.sg);
1671 iter.dma = sg_dma_address(iter.sg);
1672 iter.max = iter.dma + iter.sg->length;
1675 if (++act_pte == GEN6_PTES) {
1676 kunmap_atomic(vaddr);
1677 vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));
1681 kunmap_atomic(vaddr);
1683 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1686 static int gen6_alloc_va_range(struct i915_address_space *vm,
1687 u64 start, u64 length)
1689 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1690 struct i915_page_directory * const pd = ppgtt->base.pd;
1691 struct i915_page_table *pt, *alloc = NULL;
1692 intel_wakeref_t wakeref;
1698 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1700 spin_lock(&pd->lock);
1701 gen6_for_each_pde(pt, pd, start, length, pde) {
1702 const unsigned int count = gen6_pte_count(start, length);
1704 if (px_base(pt) == px_base(&vm->scratch[1])) {
1705 spin_unlock(&pd->lock);
1707 pt = fetch_and_zero(&alloc);
1715 fill32_px(pt, vm->scratch[0].encode);
1717 spin_lock(&pd->lock);
1718 if (pd->entry[pde] == &vm->scratch[1]) {
1719 pd->entry[pde] = pt;
1720 if (i915_vma_is_bound(ppgtt->vma,
1721 I915_VMA_GLOBAL_BIND)) {
1722 gen6_write_pde(ppgtt, pde, pt);
1727 pt = pd->entry[pde];
1731 atomic_add(count, &pt->used);
1733 spin_unlock(&pd->lock);
1736 gen6_ggtt_invalidate(vm->gt->ggtt);
1741 gen6_ppgtt_clear_range(vm, from, start - from);
1745 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
1749 static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
1751 struct i915_address_space * const vm = &ppgtt->base.vm;
1752 struct i915_page_directory * const pd = ppgtt->base.pd;
1755 ret = setup_scratch_page(vm, __GFP_HIGHMEM);
1759 vm->scratch[0].encode =
1760 vm->pte_encode(px_dma(&vm->scratch[0]),
1761 I915_CACHE_NONE, PTE_READ_ONLY);
1763 if (unlikely(setup_page_dma(vm, px_base(&vm->scratch[1])))) {
1764 cleanup_scratch_page(vm);
1768 fill32_px(&vm->scratch[1], vm->scratch[0].encode);
1769 memset_p(pd->entry, &vm->scratch[1], I915_PDES);
1774 static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
1776 struct i915_page_directory * const pd = ppgtt->base.pd;
1777 struct i915_page_dma * const scratch =
1778 px_base(&ppgtt->base.vm.scratch[1]);
1779 struct i915_page_table *pt;
1782 gen6_for_all_pdes(pt, pd, pde)
1783 if (px_base(pt) != scratch)
1784 free_px(&ppgtt->base.vm, pt);
1787 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1789 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1791 i915_vma_destroy(ppgtt->vma);
1793 gen6_ppgtt_free_pd(ppgtt);
1796 mutex_destroy(&ppgtt->pin_mutex);
1797 kfree(ppgtt->base.pd);
1800 static int pd_vma_set_pages(struct i915_vma *vma)
1802 vma->pages = ERR_PTR(-ENODEV);
1806 static void pd_vma_clear_pages(struct i915_vma *vma)
1808 GEM_BUG_ON(!vma->pages);
1813 static int pd_vma_bind(struct i915_vma *vma,
1814 enum i915_cache_level cache_level,
1817 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
1818 struct gen6_ppgtt *ppgtt = vma->private;
1819 u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE;
1820 struct i915_page_table *pt;
1823 px_base(ppgtt->base.pd)->ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);
1824 ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
1826 gen6_for_all_pdes(pt, ppgtt->base.pd, pde)
1827 gen6_write_pde(ppgtt, pde, pt);
1829 gen6_ggtt_invalidate(ggtt);
1834 static void pd_vma_unbind(struct i915_vma *vma)
1836 struct gen6_ppgtt *ppgtt = vma->private;
1837 struct i915_page_directory * const pd = ppgtt->base.pd;
1838 struct i915_page_dma * const scratch =
1839 px_base(&ppgtt->base.vm.scratch[1]);
1840 struct i915_page_table *pt;
1843 if (!ppgtt->scan_for_unused_pt)
1846 /* Free all no longer used page tables */
1847 gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
1848 if (px_base(pt) == scratch || atomic_read(&pt->used))
1851 free_px(&ppgtt->base.vm, pt);
1852 pd->entry[pde] = scratch;
1855 ppgtt->scan_for_unused_pt = false;
1858 static const struct i915_vma_ops pd_vma_ops = {
1859 .set_pages = pd_vma_set_pages,
1860 .clear_pages = pd_vma_clear_pages,
1861 .bind_vma = pd_vma_bind,
1862 .unbind_vma = pd_vma_unbind,
1865 static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
1867 struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt;
1868 struct i915_vma *vma;
1870 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
1871 GEM_BUG_ON(size > ggtt->vm.total);
1873 vma = i915_vma_alloc();
1875 return ERR_PTR(-ENOMEM);
1877 i915_active_init(&vma->active, NULL, NULL);
1879 mutex_init(&vma->pages_mutex);
1880 vma->vm = i915_vm_get(&ggtt->vm);
1881 vma->ops = &pd_vma_ops;
1882 vma->private = ppgtt;
1885 vma->fence_size = size;
1886 atomic_set(&vma->flags, I915_VMA_GGTT);
1887 vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */
1889 INIT_LIST_HEAD(&vma->obj_link);
1890 INIT_LIST_HEAD(&vma->closed_link);
1895 int gen6_ppgtt_pin(struct i915_ppgtt *base)
1897 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
1900 GEM_BUG_ON(!atomic_read(&ppgtt->base.vm.open));
1903 * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
1904 * which will be pinned into every active context.
1905 * (When vma->pin_count becomes atomic, I expect we will naturally
1906 * need a larger, unpacked, type and kill this redundancy.)
1908 if (atomic_add_unless(&ppgtt->pin_count, 1, 0))
1911 if (mutex_lock_interruptible(&ppgtt->pin_mutex))
1915 * PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1916 * allocator works in address space sizes, so it's multiplied by page
1917 * size. We allocate at the top of the GTT to avoid fragmentation.
1919 if (!atomic_read(&ppgtt->pin_count)) {
1920 err = i915_vma_pin(ppgtt->vma,
1922 PIN_GLOBAL | PIN_HIGH);
1925 atomic_inc(&ppgtt->pin_count);
1926 mutex_unlock(&ppgtt->pin_mutex);
1931 void gen6_ppgtt_unpin(struct i915_ppgtt *base)
1933 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
1935 GEM_BUG_ON(!atomic_read(&ppgtt->pin_count));
1936 if (atomic_dec_and_test(&ppgtt->pin_count))
1937 i915_vma_unpin(ppgtt->vma);
1940 void gen6_ppgtt_unpin_all(struct i915_ppgtt *base)
1942 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
1944 if (!atomic_read(&ppgtt->pin_count))
1947 i915_vma_unpin(ppgtt->vma);
1948 atomic_set(&ppgtt->pin_count, 0);
1951 static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
1953 struct i915_ggtt * const ggtt = &i915->ggtt;
1954 struct gen6_ppgtt *ppgtt;
1957 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1959 return ERR_PTR(-ENOMEM);
1961 mutex_init(&ppgtt->pin_mutex);
1963 ppgtt_init(&ppgtt->base, &i915->gt);
1964 ppgtt->base.vm.top = 1;
1966 ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;
1967 ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
1968 ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
1969 ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
1970 ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
1972 ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
1974 ppgtt->base.pd = __alloc_pd(sizeof(*ppgtt->base.pd));
1975 if (!ppgtt->base.pd) {
1980 err = gen6_ppgtt_init_scratch(ppgtt);
1984 ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
1985 if (IS_ERR(ppgtt->vma)) {
1986 err = PTR_ERR(ppgtt->vma);
1990 return &ppgtt->base;
1993 free_scratch(&ppgtt->base.vm);
1995 kfree(ppgtt->base.pd);
1998 return ERR_PTR(err);
2001 static void gtt_write_workarounds(struct intel_gt *gt)
2003 struct drm_i915_private *i915 = gt->i915;
2004 struct intel_uncore *uncore = gt->uncore;
2006 /* This function is for gtt related workarounds. This function is
2007 * called on driver load and after a GPU reset, so you can place
2008 * workarounds here even if they get overwritten by GPU reset.
2010 /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */
2011 if (IS_BROADWELL(i915))
2012 intel_uncore_write(uncore,
2013 GEN8_L3_LRA_1_GPGPU,
2014 GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2015 else if (IS_CHERRYVIEW(i915))
2016 intel_uncore_write(uncore,
2017 GEN8_L3_LRA_1_GPGPU,
2018 GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2019 else if (IS_GEN9_LP(i915))
2020 intel_uncore_write(uncore,
2021 GEN8_L3_LRA_1_GPGPU,
2022 GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2023 else if (INTEL_GEN(i915) >= 9 && INTEL_GEN(i915) <= 11)
2024 intel_uncore_write(uncore,
2025 GEN8_L3_LRA_1_GPGPU,
2026 GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2029 * To support 64K PTEs we need to first enable the use of the
2030 * Intermediate-Page-Size(IPS) bit of the PDE field via some magical
2031 * mmio, otherwise the page-walker will simply ignore the IPS bit. This
2032 * shouldn't be needed after GEN10.
2034 * 64K pages were first introduced from BDW+, although technically they
2035 * only *work* from gen9+. For pre-BDW we instead have the option for
2036 * 32K pages, but we don't currently have any support for it in our
2039 if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K) &&
2040 INTEL_GEN(i915) <= 10)
2041 intel_uncore_rmw(uncore,
2042 GEN8_GAMW_ECO_DEV_RW_IA,
2044 GAMW_ECO_ENABLE_64K_IPS_FIELD);
2046 if (IS_GEN_RANGE(i915, 8, 11)) {
2047 bool can_use_gtt_cache = true;
2050 * According to the BSpec if we use 2M/1G pages then we also
2051 * need to disable the GTT cache. At least on BDW we can see
2052 * visual corruption when using 2M pages, and not disabling the
2055 if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_2M))
2056 can_use_gtt_cache = false;
2058 /* WaGttCachingOffByDefault */
2059 intel_uncore_write(uncore,
2061 can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
2062 WARN_ON_ONCE(can_use_gtt_cache &&
2063 intel_uncore_read(uncore,
2064 HSW_GTT_CACHE_EN) == 0);
2068 int i915_ppgtt_init_hw(struct intel_gt *gt)
2070 struct drm_i915_private *i915 = gt->i915;
2072 gtt_write_workarounds(gt);
2074 if (IS_GEN(i915, 6))
2075 gen6_ppgtt_enable(gt);
2076 else if (IS_GEN(i915, 7))
2077 gen7_ppgtt_enable(gt);
2082 static struct i915_ppgtt *
2083 __ppgtt_create(struct drm_i915_private *i915)
2085 if (INTEL_GEN(i915) < 8)
2086 return gen6_ppgtt_create(i915);
2088 return gen8_ppgtt_create(i915);
2092 i915_ppgtt_create(struct drm_i915_private *i915)
2094 struct i915_ppgtt *ppgtt;
2096 ppgtt = __ppgtt_create(i915);
2100 trace_i915_ppgtt_create(&ppgtt->vm);
2105 /* Certain Gen5 chipsets require require idling the GPU before
2106 * unmapping anything from the GTT when VT-d is enabled.
2108 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2110 /* Query intel_iommu to see if we need the workaround. Presumably that
2113 return IS_GEN(dev_priv, 5) && IS_MOBILE(dev_priv) && intel_vtd_active();
2116 static void ggtt_suspend_mappings(struct i915_ggtt *ggtt)
2118 struct drm_i915_private *i915 = ggtt->vm.i915;
2120 /* Don't bother messing with faults pre GEN6 as we have little
2121 * documentation supporting that it's a good idea.
2123 if (INTEL_GEN(i915) < 6)
2126 intel_gt_check_and_clear_faults(ggtt->vm.gt);
2128 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
2130 ggtt->invalidate(ggtt);
2133 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *i915)
2135 ggtt_suspend_mappings(&i915->ggtt);
2138 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2139 struct sg_table *pages)
2142 if (dma_map_sg_attrs(&obj->base.dev->pdev->dev,
2143 pages->sgl, pages->nents,
2144 PCI_DMA_BIDIRECTIONAL,
2149 * If the DMA remap fails, one cause can be that we have
2150 * too many objects pinned in a small remapping table,
2151 * such as swiotlb. Incrementally purge all other objects and
2152 * try again - if there are no more pages to remove from
2153 * the DMA remapper, i915_gem_shrink will return 0.
2155 GEM_BUG_ON(obj->mm.pages == pages);
2156 } while (i915_gem_shrink(to_i915(obj->base.dev),
2157 obj->base.size >> PAGE_SHIFT, NULL,
2159 I915_SHRINK_UNBOUND));
2164 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2169 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2172 enum i915_cache_level level,
2175 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2176 gen8_pte_t __iomem *pte =
2177 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
2179 gen8_set_pte(pte, gen8_pte_encode(addr, level, 0));
2181 ggtt->invalidate(ggtt);
2184 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2185 struct i915_vma *vma,
2186 enum i915_cache_level level,
2189 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2190 struct sgt_iter sgt_iter;
2191 gen8_pte_t __iomem *gtt_entries;
2192 const gen8_pte_t pte_encode = gen8_pte_encode(0, level, 0);
2196 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
2197 * not to allow the user to override access to a read only page.
2200 gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2201 gtt_entries += vma->node.start / I915_GTT_PAGE_SIZE;
2202 for_each_sgt_daddr(addr, sgt_iter, vma->pages)
2203 gen8_set_pte(gtt_entries++, pte_encode | addr);
2206 * We want to flush the TLBs only after we're certain all the PTE
2207 * updates have finished.
2209 ggtt->invalidate(ggtt);
2212 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2215 enum i915_cache_level level,
2218 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2219 gen6_pte_t __iomem *pte =
2220 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
2222 iowrite32(vm->pte_encode(addr, level, flags), pte);
2224 ggtt->invalidate(ggtt);
2228 * Binds an object into the global gtt with the specified cache level. The object
2229 * will be accessible to the GPU via commands whose operands reference offsets
2230 * within the global GTT as well as accessible by the GPU through the GMADR
2231 * mapped BAR (dev_priv->mm.gtt->gtt).
2233 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2234 struct i915_vma *vma,
2235 enum i915_cache_level level,
2238 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2239 gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2240 unsigned int i = vma->node.start / I915_GTT_PAGE_SIZE;
2241 struct sgt_iter iter;
2243 for_each_sgt_daddr(addr, iter, vma->pages)
2244 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2247 * We want to flush the TLBs only after we're certain all the PTE
2248 * updates have finished.
2250 ggtt->invalidate(ggtt);
2253 static void nop_clear_range(struct i915_address_space *vm,
2254 u64 start, u64 length)
2258 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2259 u64 start, u64 length)
2261 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2262 unsigned first_entry = start / I915_GTT_PAGE_SIZE;
2263 unsigned num_entries = length / I915_GTT_PAGE_SIZE;
2264 const gen8_pte_t scratch_pte = vm->scratch[0].encode;
2265 gen8_pte_t __iomem *gtt_base =
2266 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2267 const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2270 if (WARN(num_entries > max_entries,
2271 "First entry = %d; Num entries = %d (max=%d)\n",
2272 first_entry, num_entries, max_entries))
2273 num_entries = max_entries;
2275 for (i = 0; i < num_entries; i++)
2276 gen8_set_pte(>t_base[i], scratch_pte);
2279 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
2281 struct drm_i915_private *dev_priv = vm->i915;
2284 * Make sure the internal GAM fifo has been cleared of all GTT
2285 * writes before exiting stop_machine(). This guarantees that
2286 * any aperture accesses waiting to start in another process
2287 * cannot back up behind the GTT writes causing a hang.
2288 * The register can be any arbitrary GAM register.
2290 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2293 struct insert_page {
2294 struct i915_address_space *vm;
2297 enum i915_cache_level level;
2300 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
2302 struct insert_page *arg = _arg;
2304 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
2305 bxt_vtd_ggtt_wa(arg->vm);
2310 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
2313 enum i915_cache_level level,
2316 struct insert_page arg = { vm, addr, offset, level };
2318 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
2321 struct insert_entries {
2322 struct i915_address_space *vm;
2323 struct i915_vma *vma;
2324 enum i915_cache_level level;
2328 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
2330 struct insert_entries *arg = _arg;
2332 gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags);
2333 bxt_vtd_ggtt_wa(arg->vm);
2338 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2339 struct i915_vma *vma,
2340 enum i915_cache_level level,
2343 struct insert_entries arg = { vm, vma, level, flags };
2345 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
2348 struct clear_range {
2349 struct i915_address_space *vm;
2354 static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
2356 struct clear_range *arg = _arg;
2358 gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
2359 bxt_vtd_ggtt_wa(arg->vm);
2364 static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm,
2368 struct clear_range arg = { vm, start, length };
2370 stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL);
2373 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2374 u64 start, u64 length)
2376 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2377 unsigned first_entry = start / I915_GTT_PAGE_SIZE;
2378 unsigned num_entries = length / I915_GTT_PAGE_SIZE;
2379 gen6_pte_t scratch_pte, __iomem *gtt_base =
2380 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2381 const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2384 if (WARN(num_entries > max_entries,
2385 "First entry = %d; Num entries = %d (max=%d)\n",
2386 first_entry, num_entries, max_entries))
2387 num_entries = max_entries;
2389 scratch_pte = vm->scratch[0].encode;
2390 for (i = 0; i < num_entries; i++)
2391 iowrite32(scratch_pte, >t_base[i]);
2394 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2397 enum i915_cache_level cache_level,
2400 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2401 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2403 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2406 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2407 struct i915_vma *vma,
2408 enum i915_cache_level cache_level,
2411 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2412 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2414 intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
2418 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2419 u64 start, u64 length)
2421 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2424 static int ggtt_bind_vma(struct i915_vma *vma,
2425 enum i915_cache_level cache_level,
2428 struct drm_i915_private *i915 = vma->vm->i915;
2429 struct drm_i915_gem_object *obj = vma->obj;
2430 intel_wakeref_t wakeref;
2433 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
2435 if (i915_gem_object_is_readonly(obj))
2436 pte_flags |= PTE_READ_ONLY;
2438 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2439 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2441 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
2444 * Without aliasing PPGTT there's no difference between
2445 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2446 * upgrade to both bound if we bind either to avoid double-binding.
2448 atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
2453 static void ggtt_unbind_vma(struct i915_vma *vma)
2455 struct drm_i915_private *i915 = vma->vm->i915;
2456 intel_wakeref_t wakeref;
2458 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2459 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2462 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2463 enum i915_cache_level cache_level,
2466 struct drm_i915_private *i915 = vma->vm->i915;
2470 /* Currently applicable only to VLV */
2472 if (i915_gem_object_is_readonly(vma->obj))
2473 pte_flags |= PTE_READ_ONLY;
2475 if (flags & I915_VMA_LOCAL_BIND) {
2476 struct i915_ppgtt *alias = i915_vm_to_ggtt(vma->vm)->alias;
2478 if (flags & I915_VMA_ALLOC) {
2479 ret = alias->vm.allocate_va_range(&alias->vm,
2485 set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma));
2488 GEM_BUG_ON(!test_bit(I915_VMA_ALLOC_BIT,
2489 __i915_vma_flags(vma)));
2490 alias->vm.insert_entries(&alias->vm, vma,
2491 cache_level, pte_flags);
2494 if (flags & I915_VMA_GLOBAL_BIND) {
2495 intel_wakeref_t wakeref;
2497 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2498 vma->vm->insert_entries(vma->vm, vma,
2499 cache_level, pte_flags);
2506 static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2508 struct drm_i915_private *i915 = vma->vm->i915;
2510 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
2511 struct i915_address_space *vm = vma->vm;
2512 intel_wakeref_t wakeref;
2514 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2515 vm->clear_range(vm, vma->node.start, vma->size);
2518 if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) {
2519 struct i915_address_space *vm =
2520 &i915_vm_to_ggtt(vma->vm)->alias->vm;
2522 vm->clear_range(vm, vma->node.start, vma->size);
2526 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2527 struct sg_table *pages)
2529 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2530 struct device *kdev = &dev_priv->drm.pdev->dev;
2531 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2533 if (unlikely(ggtt->do_idle_maps)) {
2534 /* XXX This does not prevent more requests being submitted! */
2535 if (intel_gt_retire_requests_timeout(ggtt->vm.gt,
2536 -MAX_SCHEDULE_TIMEOUT)) {
2537 DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2538 /* Wait a bit, in hopes it avoids the hang */
2543 dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2546 static int ggtt_set_pages(struct i915_vma *vma)
2550 GEM_BUG_ON(vma->pages);
2552 ret = i915_get_ggtt_vma_pages(vma);
2556 vma->page_sizes = vma->obj->mm.page_sizes;
2561 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
2562 unsigned long color,
2566 if (i915_node_color_differs(node, color))
2567 *start += I915_GTT_PAGE_SIZE;
2569 /* Also leave a space between the unallocated reserved node after the
2570 * GTT and any objects within the GTT, i.e. we use the color adjustment
2571 * to insert a guard page to prevent prefetches crossing over the
2574 node = list_next_entry(node, node_list);
2575 if (node->color != color)
2576 *end -= I915_GTT_PAGE_SIZE;
2579 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
2581 struct i915_ppgtt *ppgtt;
2584 ppgtt = i915_ppgtt_create(ggtt->vm.i915);
2586 return PTR_ERR(ppgtt);
2588 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
2594 * Note we only pre-allocate as far as the end of the global
2595 * GTT. On 48b / 4-level page-tables, the difference is very,
2596 * very significant! We have to preallocate as GVT/vgpu does
2597 * not like the page directory disappearing.
2599 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total);
2603 ggtt->alias = ppgtt;
2604 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
2606 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma);
2607 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
2609 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma);
2610 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
2615 i915_vm_put(&ppgtt->vm);
2619 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
2621 struct i915_ppgtt *ppgtt;
2623 ppgtt = fetch_and_zero(&ggtt->alias);
2627 i915_vm_put(&ppgtt->vm);
2629 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma;
2630 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
2633 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
2638 if (!USES_GUC(ggtt->vm.i915))
2641 GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
2642 size = ggtt->vm.total - GUC_GGTT_TOP;
2644 ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size,
2645 GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
2648 DRM_DEBUG_DRIVER("Failed to reserve top of GGTT for GuC\n");
2653 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
2655 if (drm_mm_node_allocated(&ggtt->uc_fw))
2656 drm_mm_remove_node(&ggtt->uc_fw);
2659 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
2661 ggtt_release_guc_top(ggtt);
2662 if (drm_mm_node_allocated(&ggtt->error_capture))
2663 drm_mm_remove_node(&ggtt->error_capture);
2666 static int init_ggtt(struct i915_ggtt *ggtt)
2668 /* Let GEM Manage all of the aperture.
2670 * However, leave one page at the end still bound to the scratch page.
2671 * There are a number of places where the hardware apparently prefetches
2672 * past the end of the object, and we've seen multiple hangs with the
2673 * GPU head pointer stuck in a batchbuffer bound at the last page of the
2674 * aperture. One page should be enough to keep any prefetching inside
2677 unsigned long hole_start, hole_end;
2678 struct drm_mm_node *entry;
2682 * GuC requires all resources that we're sharing with it to be placed in
2683 * non-WOPCM memory. If GuC is not present or not in use we still need a
2684 * small bias as ring wraparound at offset 0 sometimes hangs. No idea
2687 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
2688 intel_wopcm_guc_size(&ggtt->vm.i915->wopcm));
2690 ret = intel_vgt_balloon(ggtt);
2694 if (ggtt->mappable_end) {
2695 /* Reserve a mappable slot for our lockless error capture */
2696 ret = drm_mm_insert_node_in_range(&ggtt->vm.mm, &ggtt->error_capture,
2697 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
2698 0, ggtt->mappable_end,
2705 * The upper portion of the GuC address space has a sizeable hole
2706 * (several MB) that is inaccessible by GuC. Reserve this range within
2707 * GGTT as it can comfortably hold GuC/HuC firmware images.
2709 ret = ggtt_reserve_guc_top(ggtt);
2713 /* Clear any non-preallocated blocks */
2714 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
2715 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2716 hole_start, hole_end);
2717 ggtt->vm.clear_range(&ggtt->vm, hole_start,
2718 hole_end - hole_start);
2721 /* And finally clear the reserved guard page */
2722 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
2727 cleanup_init_ggtt(ggtt);
2731 int i915_init_ggtt(struct drm_i915_private *i915)
2735 ret = init_ggtt(&i915->ggtt);
2739 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
2740 ret = init_aliasing_ppgtt(&i915->ggtt);
2742 cleanup_init_ggtt(&i915->ggtt);
2748 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
2750 struct i915_vma *vma, *vn;
2752 atomic_set(&ggtt->vm.open, 0);
2754 rcu_barrier(); /* flush the RCU'ed__i915_vm_release */
2755 flush_workqueue(ggtt->vm.i915->wq);
2757 mutex_lock(&ggtt->vm.mutex);
2759 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
2760 WARN_ON(__i915_vma_unbind(vma));
2762 if (drm_mm_node_allocated(&ggtt->error_capture))
2763 drm_mm_remove_node(&ggtt->error_capture);
2765 ggtt_release_guc_top(ggtt);
2766 intel_vgt_deballoon(ggtt);
2768 ggtt->vm.cleanup(&ggtt->vm);
2770 mutex_unlock(&ggtt->vm.mutex);
2771 i915_address_space_fini(&ggtt->vm);
2773 arch_phys_wc_del(ggtt->mtrr);
2775 if (ggtt->iomap.size)
2776 io_mapping_fini(&ggtt->iomap);
2780 * i915_ggtt_driver_release - Clean up GGTT hardware initialization
2781 * @i915: i915 device
2783 void i915_ggtt_driver_release(struct drm_i915_private *i915)
2785 struct pagevec *pvec;
2787 fini_aliasing_ppgtt(&i915->ggtt);
2789 ggtt_cleanup_hw(&i915->ggtt);
2791 pvec = &i915->mm.wc_stash.pvec;
2793 set_pages_array_wb(pvec->pages, pvec->nr);
2794 __pagevec_release(pvec);
2798 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2800 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2801 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2802 return snb_gmch_ctl << 20;
2805 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2807 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2808 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2810 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2812 #ifdef CONFIG_X86_32
2813 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
2814 if (bdw_gmch_ctl > 4)
2818 return bdw_gmch_ctl << 20;
2821 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2823 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2824 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2827 return 1 << (20 + gmch_ctrl);
2832 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2834 struct drm_i915_private *dev_priv = ggtt->vm.i915;
2835 struct pci_dev *pdev = dev_priv->drm.pdev;
2836 phys_addr_t phys_addr;
2839 /* For Modern GENs the PTEs and register space are split in the BAR */
2840 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2843 * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
2844 * will be dropped. For WC mappings in general we have 64 byte burst
2845 * writes when the WC buffer is flushed, so we can't use it, but have to
2846 * resort to an uncached mapping. The WC issue is easily caught by the
2847 * readback check when writing GTT PTE entries.
2849 if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10)
2850 ggtt->gsm = ioremap_nocache(phys_addr, size);
2852 ggtt->gsm = ioremap_wc(phys_addr, size);
2854 DRM_ERROR("Failed to map the ggtt page table\n");
2858 ret = setup_scratch_page(&ggtt->vm, GFP_DMA32);
2860 DRM_ERROR("Scratch setup failed\n");
2861 /* iounmap will also get called at remove, but meh */
2866 ggtt->vm.scratch[0].encode =
2867 ggtt->vm.pte_encode(px_dma(&ggtt->vm.scratch[0]),
2868 I915_CACHE_NONE, 0);
2873 static void tgl_setup_private_ppat(struct intel_uncore *uncore)
2875 /* TGL doesn't support LLC or AGE settings */
2876 intel_uncore_write(uncore, GEN12_PAT_INDEX(0), GEN8_PPAT_WB);
2877 intel_uncore_write(uncore, GEN12_PAT_INDEX(1), GEN8_PPAT_WC);
2878 intel_uncore_write(uncore, GEN12_PAT_INDEX(2), GEN8_PPAT_WT);
2879 intel_uncore_write(uncore, GEN12_PAT_INDEX(3), GEN8_PPAT_UC);
2880 intel_uncore_write(uncore, GEN12_PAT_INDEX(4), GEN8_PPAT_WB);
2881 intel_uncore_write(uncore, GEN12_PAT_INDEX(5), GEN8_PPAT_WB);
2882 intel_uncore_write(uncore, GEN12_PAT_INDEX(6), GEN8_PPAT_WB);
2883 intel_uncore_write(uncore, GEN12_PAT_INDEX(7), GEN8_PPAT_WB);
2886 static void cnl_setup_private_ppat(struct intel_uncore *uncore)
2888 intel_uncore_write(uncore,
2890 GEN8_PPAT_WB | GEN8_PPAT_LLC);
2891 intel_uncore_write(uncore,
2893 GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
2894 intel_uncore_write(uncore,
2896 GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
2897 intel_uncore_write(uncore,
2900 intel_uncore_write(uncore,
2902 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
2903 intel_uncore_write(uncore,
2905 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
2906 intel_uncore_write(uncore,
2908 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
2909 intel_uncore_write(uncore,
2911 GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2914 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2915 * bits. When using advanced contexts each context stores its own PAT, but
2916 * writing this data shouldn't be harmful even in those cases. */
2917 static void bdw_setup_private_ppat(struct intel_uncore *uncore)
2921 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
2922 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2923 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2924 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
2925 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2926 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2927 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2928 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2930 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
2931 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
2934 static void chv_setup_private_ppat(struct intel_uncore *uncore)
2939 * Map WB on BDW to snooped on CHV.
2941 * Only the snoop bit has meaning for CHV, the rest is
2944 * The hardware will never snoop for certain types of accesses:
2945 * - CPU GTT (GMADR->GGTT->no snoop->memory)
2946 * - PPGTT page tables
2947 * - some other special cycles
2949 * As with BDW, we also need to consider the following for GT accesses:
2950 * "For GGTT, there is NO pat_sel[2:0] from the entry,
2951 * so RTL will always use the value corresponding to
2953 * Which means we must set the snoop bit in PAT entry 0
2954 * in order to keep the global status page working.
2957 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2961 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2962 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2963 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2964 GEN8_PPAT(7, CHV_PPAT_SNOOP);
2966 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
2967 intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
2970 static void gen6_gmch_remove(struct i915_address_space *vm)
2972 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2975 cleanup_scratch_page(vm);
2978 static void setup_private_pat(struct intel_uncore *uncore)
2980 struct drm_i915_private *i915 = uncore->i915;
2982 GEM_BUG_ON(INTEL_GEN(i915) < 8);
2984 if (INTEL_GEN(i915) >= 12)
2985 tgl_setup_private_ppat(uncore);
2986 else if (INTEL_GEN(i915) >= 10)
2987 cnl_setup_private_ppat(uncore);
2988 else if (IS_CHERRYVIEW(i915) || IS_GEN9_LP(i915))
2989 chv_setup_private_ppat(uncore);
2991 bdw_setup_private_ppat(uncore);
2994 static struct resource pci_resource(struct pci_dev *pdev, int bar)
2996 return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar),
2997 pci_resource_len(pdev, bar));
3000 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3002 struct drm_i915_private *dev_priv = ggtt->vm.i915;
3003 struct pci_dev *pdev = dev_priv->drm.pdev;
3008 /* TODO: We're not aware of mappable constraints on gen8 yet */
3009 if (!IS_DGFX(dev_priv)) {
3010 ggtt->gmadr = pci_resource(pdev, 2);
3011 ggtt->mappable_end = resource_size(&ggtt->gmadr);
3014 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
3016 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3018 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3020 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3021 if (IS_CHERRYVIEW(dev_priv))
3022 size = chv_get_total_gtt_size(snb_gmch_ctl);
3024 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3026 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
3027 ggtt->vm.cleanup = gen6_gmch_remove;
3028 ggtt->vm.insert_page = gen8_ggtt_insert_page;
3029 ggtt->vm.clear_range = nop_clear_range;
3030 if (intel_scanout_needs_vtd_wa(dev_priv))
3031 ggtt->vm.clear_range = gen8_ggtt_clear_range;
3033 ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
3035 /* Serialize GTT updates with aperture access on BXT if VT-d is on. */
3036 if (intel_ggtt_update_needs_vtd_wa(dev_priv) ||
3037 IS_CHERRYVIEW(dev_priv) /* fails with concurrent use/update */) {
3038 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
3039 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL;
3040 if (ggtt->vm.clear_range != nop_clear_range)
3041 ggtt->vm.clear_range = bxt_vtd_ggtt_clear_range__BKL;
3044 ggtt->invalidate = gen6_ggtt_invalidate;
3046 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma;
3047 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
3048 ggtt->vm.vma_ops.set_pages = ggtt_set_pages;
3049 ggtt->vm.vma_ops.clear_pages = clear_pages;
3051 ggtt->vm.pte_encode = gen8_pte_encode;
3053 setup_private_pat(ggtt->vm.gt->uncore);
3055 return ggtt_probe_common(ggtt, size);
3058 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3060 struct drm_i915_private *dev_priv = ggtt->vm.i915;
3061 struct pci_dev *pdev = dev_priv->drm.pdev;
3067 (struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
3068 pci_resource_len(pdev, 2));
3069 ggtt->mappable_end = resource_size(&ggtt->gmadr);
3071 /* 64/512MB is the current min/max we actually know of, but this is just
3072 * a coarse sanity check.
3074 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3075 DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end);
3079 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40));
3081 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3083 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3084 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3086 size = gen6_get_total_gtt_size(snb_gmch_ctl);
3087 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
3089 ggtt->vm.clear_range = nop_clear_range;
3090 if (!HAS_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3091 ggtt->vm.clear_range = gen6_ggtt_clear_range;
3092 ggtt->vm.insert_page = gen6_ggtt_insert_page;
3093 ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
3094 ggtt->vm.cleanup = gen6_gmch_remove;
3096 ggtt->invalidate = gen6_ggtt_invalidate;
3098 if (HAS_EDRAM(dev_priv))
3099 ggtt->vm.pte_encode = iris_pte_encode;
3100 else if (IS_HASWELL(dev_priv))
3101 ggtt->vm.pte_encode = hsw_pte_encode;
3102 else if (IS_VALLEYVIEW(dev_priv))
3103 ggtt->vm.pte_encode = byt_pte_encode;
3104 else if (INTEL_GEN(dev_priv) >= 7)
3105 ggtt->vm.pte_encode = ivb_pte_encode;
3107 ggtt->vm.pte_encode = snb_pte_encode;
3109 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma;
3110 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
3111 ggtt->vm.vma_ops.set_pages = ggtt_set_pages;
3112 ggtt->vm.vma_ops.clear_pages = clear_pages;
3114 return ggtt_probe_common(ggtt, size);
3117 static void i915_gmch_remove(struct i915_address_space *vm)
3119 intel_gmch_remove();
3122 static int i915_gmch_probe(struct i915_ggtt *ggtt)
3124 struct drm_i915_private *dev_priv = ggtt->vm.i915;
3125 phys_addr_t gmadr_base;
3128 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3130 DRM_ERROR("failed to set up gmch\n");
3134 intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end);
3137 (struct resource) DEFINE_RES_MEM(gmadr_base,
3138 ggtt->mappable_end);
3140 ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3141 ggtt->vm.insert_page = i915_ggtt_insert_page;
3142 ggtt->vm.insert_entries = i915_ggtt_insert_entries;
3143 ggtt->vm.clear_range = i915_ggtt_clear_range;
3144 ggtt->vm.cleanup = i915_gmch_remove;
3146 ggtt->invalidate = gmch_ggtt_invalidate;
3148 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma;
3149 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
3150 ggtt->vm.vma_ops.set_pages = ggtt_set_pages;
3151 ggtt->vm.vma_ops.clear_pages = clear_pages;
3153 if (unlikely(ggtt->do_idle_maps))
3154 dev_notice(dev_priv->drm.dev,
3155 "Applying Ironlake quirks for intel_iommu\n");
3160 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
3162 struct drm_i915_private *i915 = gt->i915;
3166 ggtt->vm.i915 = i915;
3167 ggtt->vm.dma = &i915->drm.pdev->dev;
3169 if (INTEL_GEN(i915) <= 5)
3170 ret = i915_gmch_probe(ggtt);
3171 else if (INTEL_GEN(i915) < 8)
3172 ret = gen6_gmch_probe(ggtt);
3174 ret = gen8_gmch_probe(ggtt);
3178 if ((ggtt->vm.total - 1) >> 32) {
3179 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3180 " of address space! Found %lldM!\n",
3181 ggtt->vm.total >> 20);
3182 ggtt->vm.total = 1ULL << 32;
3183 ggtt->mappable_end =
3184 min_t(u64, ggtt->mappable_end, ggtt->vm.total);
3187 if (ggtt->mappable_end > ggtt->vm.total) {
3188 DRM_ERROR("mappable aperture extends past end of GGTT,"
3189 " aperture=%pa, total=%llx\n",
3190 &ggtt->mappable_end, ggtt->vm.total);
3191 ggtt->mappable_end = ggtt->vm.total;
3194 /* GMADR is the PCI mmio aperture into the global GTT. */
3195 DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->vm.total >> 20);
3196 DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20);
3197 DRM_DEBUG_DRIVER("DSM size = %lluM\n",
3198 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
3204 * i915_ggtt_probe_hw - Probe GGTT hardware location
3205 * @i915: i915 device
3207 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
3211 ret = ggtt_probe_hw(&i915->ggtt, &i915->gt);
3215 if (intel_vtd_active())
3216 dev_info(i915->drm.dev, "VT-d active for gfx access\n");
3221 static int ggtt_init_hw(struct i915_ggtt *ggtt)
3223 struct drm_i915_private *i915 = ggtt->vm.i915;
3225 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
3227 ggtt->vm.is_ggtt = true;
3229 /* Only VLV supports read-only GGTT mappings */
3230 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
3232 if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
3233 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
3235 if (ggtt->mappable_end) {
3236 if (!io_mapping_init_wc(&ggtt->iomap,
3238 ggtt->mappable_end)) {
3239 ggtt->vm.cleanup(&ggtt->vm);
3243 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
3244 ggtt->mappable_end);
3247 i915_ggtt_init_fences(ggtt);
3253 * i915_ggtt_init_hw - Initialize GGTT hardware
3254 * @dev_priv: i915 device
3256 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3260 stash_init(&dev_priv->mm.wc_stash);
3262 /* Note that we use page colouring to enforce a guard page at the
3263 * end of the address space. This is required as the CS may prefetch
3264 * beyond the end of the batch buffer, across the page boundary,
3265 * and beyond the end of the GTT if we do not provide a guard.
3267 ret = ggtt_init_hw(&dev_priv->ggtt);
3274 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3276 if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3282 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt)
3284 GEM_BUG_ON(ggtt->invalidate != gen6_ggtt_invalidate);
3286 ggtt->invalidate = guc_ggtt_invalidate;
3288 ggtt->invalidate(ggtt);
3291 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt)
3293 /* XXX Temporary pardon for error unload */
3294 if (ggtt->invalidate == gen6_ggtt_invalidate)
3297 /* We should only be called after i915_ggtt_enable_guc() */
3298 GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate);
3300 ggtt->invalidate = gen6_ggtt_invalidate;
3302 ggtt->invalidate(ggtt);
3305 static void ggtt_restore_mappings(struct i915_ggtt *ggtt)
3307 struct i915_vma *vma, *vn;
3311 intel_gt_check_and_clear_faults(ggtt->vm.gt);
3313 mutex_lock(&ggtt->vm.mutex);
3315 /* First fill our portion of the GTT with scratch pages */
3316 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
3318 /* Skip rewriting PTE on VMA unbind. */
3319 open = atomic_xchg(&ggtt->vm.open, 0);
3321 /* clflush objects bound into the GGTT and rebind them. */
3322 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
3323 struct drm_i915_gem_object *obj = vma->obj;
3325 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
3328 if (!__i915_vma_unbind(vma))
3331 clear_bit(I915_VMA_GLOBAL_BIND_BIT, __i915_vma_flags(vma));
3332 WARN_ON(i915_vma_bind(vma,
3333 obj ? obj->cache_level : 0,
3335 if (obj) { /* only used during resume => exclusive access */
3336 flush |= fetch_and_zero(&obj->write_domain);
3337 obj->read_domains |= I915_GEM_DOMAIN_GTT;
3341 atomic_set(&ggtt->vm.open, open);
3342 ggtt->invalidate(ggtt);
3344 mutex_unlock(&ggtt->vm.mutex);
3347 wbinvd_on_all_cpus();
3350 void i915_gem_restore_gtt_mappings(struct drm_i915_private *i915)
3352 struct i915_ggtt *ggtt = &i915->ggtt;
3354 ggtt_restore_mappings(ggtt);
3356 if (INTEL_GEN(i915) >= 8)
3357 setup_private_pat(ggtt->vm.gt->uncore);
3360 static struct scatterlist *
3361 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
3362 unsigned int width, unsigned int height,
3363 unsigned int stride,
3364 struct sg_table *st, struct scatterlist *sg)
3366 unsigned int column, row;
3367 unsigned int src_idx;
3369 for (column = 0; column < width; column++) {
3370 src_idx = stride * (height - 1) + column + offset;
3371 for (row = 0; row < height; row++) {
3373 /* We don't need the pages, but need to initialize
3374 * the entries so the sg list can be happily traversed.
3375 * The only thing we need are DMA addresses.
3377 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
3378 sg_dma_address(sg) =
3379 i915_gem_object_get_dma_address(obj, src_idx);
3380 sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
3389 static noinline struct sg_table *
3390 intel_rotate_pages(struct intel_rotation_info *rot_info,
3391 struct drm_i915_gem_object *obj)
3393 unsigned int size = intel_rotation_info_size(rot_info);
3394 struct sg_table *st;
3395 struct scatterlist *sg;
3399 /* Allocate target SG list. */
3400 st = kmalloc(sizeof(*st), GFP_KERNEL);
3404 ret = sg_alloc_table(st, size, GFP_KERNEL);
3411 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3412 sg = rotate_pages(obj, rot_info->plane[i].offset,
3413 rot_info->plane[i].width, rot_info->plane[i].height,
3414 rot_info->plane[i].stride, st, sg);
3423 DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3424 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3426 return ERR_PTR(ret);
3429 static struct scatterlist *
3430 remap_pages(struct drm_i915_gem_object *obj, unsigned int offset,
3431 unsigned int width, unsigned int height,
3432 unsigned int stride,
3433 struct sg_table *st, struct scatterlist *sg)
3437 for (row = 0; row < height; row++) {
3438 unsigned int left = width * I915_GTT_PAGE_SIZE;
3442 unsigned int length;
3444 /* We don't need the pages, but need to initialize
3445 * the entries so the sg list can be happily traversed.
3446 * The only thing we need are DMA addresses.
3449 addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
3451 length = min(left, length);
3455 sg_set_page(sg, NULL, length, 0);
3456 sg_dma_address(sg) = addr;
3457 sg_dma_len(sg) = length;
3460 offset += length / I915_GTT_PAGE_SIZE;
3464 offset += stride - width;
3470 static noinline struct sg_table *
3471 intel_remap_pages(struct intel_remapped_info *rem_info,
3472 struct drm_i915_gem_object *obj)
3474 unsigned int size = intel_remapped_info_size(rem_info);
3475 struct sg_table *st;
3476 struct scatterlist *sg;
3480 /* Allocate target SG list. */
3481 st = kmalloc(sizeof(*st), GFP_KERNEL);
3485 ret = sg_alloc_table(st, size, GFP_KERNEL);
3492 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
3493 sg = remap_pages(obj, rem_info->plane[i].offset,
3494 rem_info->plane[i].width, rem_info->plane[i].height,
3495 rem_info->plane[i].stride, st, sg);
3506 DRM_DEBUG_DRIVER("Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3507 obj->base.size, rem_info->plane[0].width, rem_info->plane[0].height, size);
3509 return ERR_PTR(ret);
3512 static noinline struct sg_table *
3513 intel_partial_pages(const struct i915_ggtt_view *view,
3514 struct drm_i915_gem_object *obj)
3516 struct sg_table *st;
3517 struct scatterlist *sg, *iter;
3518 unsigned int count = view->partial.size;
3519 unsigned int offset;
3522 st = kmalloc(sizeof(*st), GFP_KERNEL);
3526 ret = sg_alloc_table(st, count, GFP_KERNEL);
3530 iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
3538 len = min(iter->length - (offset << PAGE_SHIFT),
3539 count << PAGE_SHIFT);
3540 sg_set_page(sg, NULL, len, 0);
3541 sg_dma_address(sg) =
3542 sg_dma_address(iter) + (offset << PAGE_SHIFT);
3543 sg_dma_len(sg) = len;
3546 count -= len >> PAGE_SHIFT;
3549 i915_sg_trim(st); /* Drop any unused tail entries. */
3555 iter = __sg_next(iter);
3562 return ERR_PTR(ret);
3566 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3570 /* The vma->pages are only valid within the lifespan of the borrowed
3571 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3572 * must be the vma->pages. A simple rule is that vma->pages must only
3573 * be accessed when the obj->mm.pages are pinned.
3575 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3577 switch (vma->ggtt_view.type) {
3579 GEM_BUG_ON(vma->ggtt_view.type);
3581 case I915_GGTT_VIEW_NORMAL:
3582 vma->pages = vma->obj->mm.pages;
3585 case I915_GGTT_VIEW_ROTATED:
3587 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
3590 case I915_GGTT_VIEW_REMAPPED:
3592 intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
3595 case I915_GGTT_VIEW_PARTIAL:
3596 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3601 if (IS_ERR(vma->pages)) {
3602 ret = PTR_ERR(vma->pages);
3604 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3605 vma->ggtt_view.type, ret);
3611 * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
3612 * @vm: the &struct i915_address_space
3613 * @node: the &struct drm_mm_node (typically i915_vma.mode)
3614 * @size: how much space to allocate inside the GTT,
3615 * must be #I915_GTT_PAGE_SIZE aligned
3616 * @offset: where to insert inside the GTT,
3617 * must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
3618 * (@offset + @size) must fit within the address space
3619 * @color: color to apply to node, if this node is not from a VMA,
3620 * color must be #I915_COLOR_UNEVICTABLE
3621 * @flags: control search and eviction behaviour
3623 * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
3624 * the address space (using @size and @color). If the @node does not fit, it
3625 * tries to evict any overlapping nodes from the GTT, including any
3626 * neighbouring nodes if the colors do not match (to ensure guard pages between
3627 * differing domains). See i915_gem_evict_for_node() for the gory details
3628 * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
3629 * evicting active overlapping objects, and any overlapping node that is pinned
3630 * or marked as unevictable will also result in failure.
3632 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3633 * asked to wait for eviction and interrupted.
3635 int i915_gem_gtt_reserve(struct i915_address_space *vm,
3636 struct drm_mm_node *node,
3637 u64 size, u64 offset, unsigned long color,
3643 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3644 GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
3645 GEM_BUG_ON(range_overflows(offset, size, vm->total));
3646 GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm);
3647 GEM_BUG_ON(drm_mm_node_allocated(node));
3650 node->start = offset;
3651 node->color = color;
3653 err = drm_mm_reserve_node(&vm->mm, node);
3657 if (flags & PIN_NOEVICT)
3660 err = i915_gem_evict_for_node(vm, node, flags);
3662 err = drm_mm_reserve_node(&vm->mm, node);
3667 static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
3671 GEM_BUG_ON(range_overflows(start, len, end));
3672 GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
3674 range = round_down(end - len, align) - round_up(start, align);
3676 if (sizeof(unsigned long) == sizeof(u64)) {
3677 addr = get_random_long();
3679 addr = get_random_int();
3680 if (range > U32_MAX) {
3682 addr |= get_random_int();
3685 div64_u64_rem(addr, range, &addr);
3689 return round_up(start, align);
3693 * i915_gem_gtt_insert - insert a node into an address_space (GTT)
3694 * @vm: the &struct i915_address_space
3695 * @node: the &struct drm_mm_node (typically i915_vma.node)
3696 * @size: how much space to allocate inside the GTT,
3697 * must be #I915_GTT_PAGE_SIZE aligned
3698 * @alignment: required alignment of starting offset, may be 0 but
3699 * if specified, this must be a power-of-two and at least
3700 * #I915_GTT_MIN_ALIGNMENT
3701 * @color: color to apply to node
3702 * @start: start of any range restriction inside GTT (0 for all),
3703 * must be #I915_GTT_PAGE_SIZE aligned
3704 * @end: end of any range restriction inside GTT (U64_MAX for all),
3705 * must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
3706 * @flags: control search and eviction behaviour
3708 * i915_gem_gtt_insert() first searches for an available hole into which
3709 * is can insert the node. The hole address is aligned to @alignment and
3710 * its @size must then fit entirely within the [@start, @end] bounds. The
3711 * nodes on either side of the hole must match @color, or else a guard page
3712 * will be inserted between the two nodes (or the node evicted). If no
3713 * suitable hole is found, first a victim is randomly selected and tested
3714 * for eviction, otherwise then the LRU list of objects within the GTT
3715 * is scanned to find the first set of replacement nodes to create the hole.
3716 * Those old overlapping nodes are evicted from the GTT (and so must be
3717 * rebound before any future use). Any node that is currently pinned cannot
3718 * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
3719 * active and #PIN_NONBLOCK is specified, that node is also skipped when
3720 * searching for an eviction candidate. See i915_gem_evict_something() for
3721 * the gory details on the eviction algorithm.
3723 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3724 * asked to wait for eviction and interrupted.
3726 int i915_gem_gtt_insert(struct i915_address_space *vm,
3727 struct drm_mm_node *node,
3728 u64 size, u64 alignment, unsigned long color,
3729 u64 start, u64 end, unsigned int flags)
3731 enum drm_mm_insert_mode mode;
3735 lockdep_assert_held(&vm->mutex);
3738 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3739 GEM_BUG_ON(alignment && !is_power_of_2(alignment));
3740 GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
3741 GEM_BUG_ON(start >= end);
3742 GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
3743 GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
3744 GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm);
3745 GEM_BUG_ON(drm_mm_node_allocated(node));
3747 if (unlikely(range_overflows(start, size, end)))
3750 if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
3753 mode = DRM_MM_INSERT_BEST;
3754 if (flags & PIN_HIGH)
3755 mode = DRM_MM_INSERT_HIGHEST;
3756 if (flags & PIN_MAPPABLE)
3757 mode = DRM_MM_INSERT_LOW;
3759 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3760 * so we know that we always have a minimum alignment of 4096.
3761 * The drm_mm range manager is optimised to return results
3762 * with zero alignment, so where possible use the optimal
3765 BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
3766 if (alignment <= I915_GTT_MIN_ALIGNMENT)
3769 err = drm_mm_insert_node_in_range(&vm->mm, node,
3770 size, alignment, color,
3775 if (mode & DRM_MM_INSERT_ONCE) {
3776 err = drm_mm_insert_node_in_range(&vm->mm, node,
3777 size, alignment, color,
3779 DRM_MM_INSERT_BEST);
3784 if (flags & PIN_NOEVICT)
3788 * No free space, pick a slot at random.
3790 * There is a pathological case here using a GTT shared between
3791 * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
3793 * |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
3794 * (64k objects) (448k objects)
3796 * Now imagine that the eviction LRU is ordered top-down (just because
3797 * pathology meets real life), and that we need to evict an object to
3798 * make room inside the aperture. The eviction scan then has to walk
3799 * the 448k list before it finds one within range. And now imagine that
3800 * it has to search for a new hole between every byte inside the memcpy,
3801 * for several simultaneous clients.
3803 * On a full-ppgtt system, if we have run out of available space, there
3804 * will be lots and lots of objects in the eviction list! Again,
3805 * searching that LRU list may be slow if we are also applying any
3806 * range restrictions (e.g. restriction to low 4GiB) and so, for
3807 * simplicity and similarilty between different GTT, try the single
3808 * random replacement first.
3810 offset = random_offset(start, end,
3811 size, alignment ?: I915_GTT_MIN_ALIGNMENT);
3812 err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
3816 if (flags & PIN_NOSEARCH)
3819 /* Randomly selected placement is pinned, do a search */
3820 err = i915_gem_evict_something(vm, size, alignment, color,
3825 return drm_mm_insert_node_in_range(&vm->mm, node,
3826 size, alignment, color,
3827 start, end, DRM_MM_INSERT_EVICT);
3830 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3831 #include "selftests/mock_gtt.c"
3832 #include "selftests/i915_gem_gtt.c"