2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include "display/intel_frontbuffer.h"
8 #include "gt/intel_gt.h"
11 #include "i915_gem_clflush.h"
12 #include "i915_gem_gtt.h"
13 #include "i915_gem_ioctls.h"
14 #include "i915_gem_object.h"
16 #include "i915_gem_lmem.h"
17 #include "i915_gem_mman.h"
19 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
21 return !(obj->cache_level == I915_CACHE_NONE ||
22 obj->cache_level == I915_CACHE_WT);
26 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
30 assert_object_held(obj);
32 if (!(obj->write_domain & flush_domains))
35 switch (obj->write_domain) {
36 case I915_GEM_DOMAIN_GTT:
37 spin_lock(&obj->vma.lock);
38 for_each_ggtt_vma(vma, obj) {
39 if (i915_vma_unset_ggtt_write(vma))
40 intel_gt_flush_ggtt_writes(vma->vm->gt);
42 spin_unlock(&obj->vma.lock);
44 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
47 case I915_GEM_DOMAIN_WC:
51 case I915_GEM_DOMAIN_CPU:
52 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
55 case I915_GEM_DOMAIN_RENDER:
56 if (gpu_write_needs_clflush(obj))
57 obj->cache_dirty = true;
61 obj->write_domain = 0;
64 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
67 * We manually flush the CPU domain so that we can override and
68 * force the flush for the display, and perform it asyncrhonously.
70 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
72 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
73 obj->write_domain = 0;
76 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
78 if (!i915_gem_object_is_framebuffer(obj))
81 i915_gem_object_lock(obj, NULL);
82 __i915_gem_object_flush_for_display(obj);
83 i915_gem_object_unlock(obj);
86 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
88 if (i915_gem_object_is_framebuffer(obj))
89 __i915_gem_object_flush_for_display(obj);
93 * Moves a single object to the WC read, and possibly write domain.
94 * @obj: object to act on
95 * @write: ask for write access or read only
97 * This function returns when the move is complete, including waiting on
101 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
105 assert_object_held(obj);
107 ret = i915_gem_object_wait(obj,
108 I915_WAIT_INTERRUPTIBLE |
109 (write ? I915_WAIT_ALL : 0),
110 MAX_SCHEDULE_TIMEOUT);
114 if (obj->write_domain == I915_GEM_DOMAIN_WC)
117 /* Flush and acquire obj->pages so that we are coherent through
118 * direct access in memory with previous cached writes through
119 * shmemfs and that our cache domain tracking remains valid.
120 * For example, if the obj->filp was moved to swap without us
121 * being notified and releasing the pages, we would mistakenly
122 * continue to assume that the obj remained out of the CPU cached
125 ret = i915_gem_object_pin_pages(obj);
129 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
131 /* Serialise direct access to this object with the barriers for
132 * coherent writes from the GPU, by effectively invalidating the
133 * WC domain upon first access.
135 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
138 /* It should now be out of any other write domains, and we can update
139 * the domain values for our changes.
141 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
142 obj->read_domains |= I915_GEM_DOMAIN_WC;
144 obj->read_domains = I915_GEM_DOMAIN_WC;
145 obj->write_domain = I915_GEM_DOMAIN_WC;
146 obj->mm.dirty = true;
149 i915_gem_object_unpin_pages(obj);
154 * Moves a single object to the GTT read, and possibly write domain.
155 * @obj: object to act on
156 * @write: ask for write access or read only
158 * This function returns when the move is complete, including waiting on
162 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
166 assert_object_held(obj);
168 ret = i915_gem_object_wait(obj,
169 I915_WAIT_INTERRUPTIBLE |
170 (write ? I915_WAIT_ALL : 0),
171 MAX_SCHEDULE_TIMEOUT);
175 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
178 /* Flush and acquire obj->pages so that we are coherent through
179 * direct access in memory with previous cached writes through
180 * shmemfs and that our cache domain tracking remains valid.
181 * For example, if the obj->filp was moved to swap without us
182 * being notified and releasing the pages, we would mistakenly
183 * continue to assume that the obj remained out of the CPU cached
186 ret = i915_gem_object_pin_pages(obj);
190 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
192 /* Serialise direct access to this object with the barriers for
193 * coherent writes from the GPU, by effectively invalidating the
194 * GTT domain upon first access.
196 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
199 /* It should now be out of any other write domains, and we can update
200 * the domain values for our changes.
202 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
203 obj->read_domains |= I915_GEM_DOMAIN_GTT;
205 struct i915_vma *vma;
207 obj->read_domains = I915_GEM_DOMAIN_GTT;
208 obj->write_domain = I915_GEM_DOMAIN_GTT;
209 obj->mm.dirty = true;
211 spin_lock(&obj->vma.lock);
212 for_each_ggtt_vma(vma, obj)
213 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
214 i915_vma_set_ggtt_write(vma);
215 spin_unlock(&obj->vma.lock);
218 i915_gem_object_unpin_pages(obj);
223 * Changes the cache-level of an object across all VMA.
224 * @obj: object to act on
225 * @cache_level: new cache level to set for the object
227 * After this function returns, the object will be in the new cache-level
228 * across all GTT and the contents of the backing storage will be coherent,
229 * with respect to the new cache-level. In order to keep the backing storage
230 * coherent for all users, we only allow a single cache level to be set
231 * globally on the object and prevent it from being changed whilst the
232 * hardware is reading from the object. That is if the object is currently
233 * on the scanout it will be set to uncached (or equivalent display
234 * cache coherency) and all non-MOCS GPU access will also be uncached so
235 * that all direct access to the scanout remains coherent.
237 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
238 enum i915_cache_level cache_level)
242 if (obj->cache_level == cache_level)
245 ret = i915_gem_object_wait(obj,
246 I915_WAIT_INTERRUPTIBLE |
248 MAX_SCHEDULE_TIMEOUT);
252 /* Always invalidate stale cachelines */
253 if (obj->cache_level != cache_level) {
254 i915_gem_object_set_cache_coherency(obj, cache_level);
255 obj->cache_dirty = true;
258 /* The cache-level will be applied when each vma is rebound. */
259 return i915_gem_object_unbind(obj,
260 I915_GEM_OBJECT_UNBIND_ACTIVE |
261 I915_GEM_OBJECT_UNBIND_BARRIER);
264 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
265 struct drm_file *file)
267 struct drm_i915_gem_caching *args = data;
268 struct drm_i915_gem_object *obj;
271 if (IS_DGFX(to_i915(dev)))
275 obj = i915_gem_object_lookup_rcu(file, args->handle);
281 switch (obj->cache_level) {
283 case I915_CACHE_L3_LLC:
284 args->caching = I915_CACHING_CACHED;
288 args->caching = I915_CACHING_DISPLAY;
292 args->caching = I915_CACHING_NONE;
300 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
301 struct drm_file *file)
303 struct drm_i915_private *i915 = to_i915(dev);
304 struct drm_i915_gem_caching *args = data;
305 struct drm_i915_gem_object *obj;
306 enum i915_cache_level level;
312 switch (args->caching) {
313 case I915_CACHING_NONE:
314 level = I915_CACHE_NONE;
316 case I915_CACHING_CACHED:
318 * Due to a HW issue on BXT A stepping, GPU stores via a
319 * snooped mapping may leave stale data in a corresponding CPU
320 * cacheline, whereas normally such cachelines would get
323 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
326 level = I915_CACHE_LLC;
328 case I915_CACHING_DISPLAY:
329 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
335 obj = i915_gem_object_lookup(file, args->handle);
340 * The caching mode of proxy object is handled by its generator, and
341 * not allowed to be changed by userspace.
343 if (i915_gem_object_is_proxy(obj)) {
345 * Silently allow cached for userptr; the vulkan driver
346 * sets all objects to cached
348 if (!i915_gem_object_is_userptr(obj) ||
349 args->caching != I915_CACHING_CACHED)
355 ret = i915_gem_object_lock_interruptible(obj, NULL);
359 ret = i915_gem_object_set_cache_level(obj, level);
360 i915_gem_object_unlock(obj);
363 i915_gem_object_put(obj);
368 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
369 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
370 * (for pageflips). We only flush the caches while preparing the buffer for
371 * display, the callers are responsible for frontbuffer flush.
374 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
375 struct i915_gem_ww_ctx *ww,
377 const struct i915_ggtt_view *view,
380 struct drm_i915_private *i915 = to_i915(obj->base.dev);
381 struct i915_vma *vma;
384 /* Frame buffer must be in LMEM */
385 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
386 return ERR_PTR(-EINVAL);
389 * The display engine is not coherent with the LLC cache on gen6. As
390 * a result, we make sure that the pinning that is about to occur is
391 * done with uncached PTEs. This is lowest common denominator for all
394 * However for gen6+, we could do better by using the GFDT bit instead
395 * of uncaching, which would allow us to flush all the LLC-cached data
396 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
398 ret = i915_gem_object_set_cache_level(obj,
400 I915_CACHE_WT : I915_CACHE_NONE);
405 * As the user may map the buffer once pinned in the display plane
406 * (e.g. libkms for the bootup splash), we have to ensure that we
407 * always use map_and_fenceable for all scanout buffers. However,
408 * it may simply be too big to fit into mappable, in which case
409 * put it anyway and hope that userspace can cope (but always first
410 * try to preserve the existing ABI).
412 vma = ERR_PTR(-ENOSPC);
413 if ((flags & PIN_MAPPABLE) == 0 &&
414 (!view || view->type == I915_GGTT_VIEW_NORMAL))
415 vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
416 flags | PIN_MAPPABLE |
418 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
419 vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,
424 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
425 i915_vma_mark_scanout(vma);
427 i915_gem_object_flush_if_display_locked(obj);
433 * Moves a single object to the CPU read, and possibly write domain.
434 * @obj: object to act on
435 * @write: requesting write or read-only access
437 * This function returns when the move is complete, including waiting on
441 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
445 assert_object_held(obj);
447 ret = i915_gem_object_wait(obj,
448 I915_WAIT_INTERRUPTIBLE |
449 (write ? I915_WAIT_ALL : 0),
450 MAX_SCHEDULE_TIMEOUT);
454 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
456 /* Flush the CPU cache if it's still invalid. */
457 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
458 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
459 obj->read_domains |= I915_GEM_DOMAIN_CPU;
462 /* It should now be out of any other write domains, and we can update
463 * the domain values for our changes.
465 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
467 /* If we're writing through the CPU, then the GPU read domains will
468 * need to be invalidated at next use.
471 __start_cpu_write(obj);
477 * Called when user space prepares to use an object with the CPU, either
478 * through the mmap ioctl's mapping or a GTT mapping.
480 * @data: ioctl data blob
484 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
485 struct drm_file *file)
487 struct drm_i915_gem_set_domain *args = data;
488 struct drm_i915_gem_object *obj;
489 u32 read_domains = args->read_domains;
490 u32 write_domain = args->write_domain;
493 /* Only handle setting domains to types used by the CPU. */
494 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
498 * Having something in the write domain implies it's in the read
499 * domain, and only that read domain. Enforce that in the request.
501 if (write_domain && read_domains != write_domain)
507 obj = i915_gem_object_lookup(file, args->handle);
512 * Try to flush the object off the GPU without holding the lock.
513 * We will repeat the flush holding the lock in the normal manner
514 * to catch cases where we are gazumped.
516 err = i915_gem_object_wait(obj,
517 I915_WAIT_INTERRUPTIBLE |
519 (write_domain ? I915_WAIT_ALL : 0),
520 MAX_SCHEDULE_TIMEOUT);
524 if (i915_gem_object_is_userptr(obj)) {
526 * Try to grab userptr pages, iris uses set_domain to check
529 err = i915_gem_object_userptr_validate(obj);
531 err = i915_gem_object_wait(obj,
532 I915_WAIT_INTERRUPTIBLE |
534 (write_domain ? I915_WAIT_ALL : 0),
535 MAX_SCHEDULE_TIMEOUT);
540 * Proxy objects do not control access to the backing storage, ergo
541 * they cannot be used as a means to manipulate the cache domain
542 * tracking for that backing storage. The proxy object is always
543 * considered to be outside of any cache domain.
545 if (i915_gem_object_is_proxy(obj)) {
550 err = i915_gem_object_lock_interruptible(obj, NULL);
555 * Flush and acquire obj->pages so that we are coherent through
556 * direct access in memory with previous cached writes through
557 * shmemfs and that our cache domain tracking remains valid.
558 * For example, if the obj->filp was moved to swap without us
559 * being notified and releasing the pages, we would mistakenly
560 * continue to assume that the obj remained out of the CPU cached
563 err = i915_gem_object_pin_pages(obj);
568 * Already in the desired write domain? Nothing for us to do!
570 * We apply a little bit of cunning here to catch a broader set of
571 * no-ops. If obj->write_domain is set, we must be in the same
572 * obj->read_domains, and only that domain. Therefore, if that
573 * obj->write_domain matches the request read_domains, we are
574 * already in the same read/write domain and can skip the operation,
575 * without having to further check the requested write_domain.
577 if (READ_ONCE(obj->write_domain) == read_domains)
580 if (read_domains & I915_GEM_DOMAIN_WC)
581 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
582 else if (read_domains & I915_GEM_DOMAIN_GTT)
583 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
585 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
588 i915_gem_object_unpin_pages(obj);
591 i915_gem_object_unlock(obj);
593 if (!err && write_domain)
594 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
597 i915_gem_object_put(obj);
602 * Pins the specified object's pages and synchronizes the object with
603 * GPU accesses. Sets needs_clflush to non-zero if the caller should
604 * flush the object from the CPU cache.
606 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
607 unsigned int *needs_clflush)
612 if (!i915_gem_object_has_struct_page(obj))
615 assert_object_held(obj);
617 ret = i915_gem_object_wait(obj,
618 I915_WAIT_INTERRUPTIBLE,
619 MAX_SCHEDULE_TIMEOUT);
623 ret = i915_gem_object_pin_pages(obj);
627 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
628 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
629 ret = i915_gem_object_set_to_cpu_domain(obj, false);
636 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
638 /* If we're not in the cpu read domain, set ourself into the gtt
639 * read domain and manually flush cachelines (if required). This
640 * optimizes for the case when the gpu will dirty the data
641 * anyway again before the next pread happens.
643 if (!obj->cache_dirty &&
644 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
645 *needs_clflush = CLFLUSH_BEFORE;
648 /* return with the pages pinned */
652 i915_gem_object_unpin_pages(obj);
656 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
657 unsigned int *needs_clflush)
662 if (!i915_gem_object_has_struct_page(obj))
665 assert_object_held(obj);
667 ret = i915_gem_object_wait(obj,
668 I915_WAIT_INTERRUPTIBLE |
670 MAX_SCHEDULE_TIMEOUT);
674 ret = i915_gem_object_pin_pages(obj);
678 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
679 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
680 ret = i915_gem_object_set_to_cpu_domain(obj, true);
687 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
689 /* If we're not in the cpu write domain, set ourself into the
690 * gtt write domain and manually flush cachelines (as required).
691 * This optimizes for the case when the gpu will use the data
692 * right away and we therefore have to clflush anyway.
694 if (!obj->cache_dirty) {
695 *needs_clflush |= CLFLUSH_AFTER;
698 * Same trick applies to invalidate partially written
699 * cachelines read before writing.
701 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
702 *needs_clflush |= CLFLUSH_BEFORE;
706 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
707 obj->mm.dirty = true;
708 /* return with the pages pinned */
712 i915_gem_object_unpin_pages(obj);