2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include "display/intel_frontbuffer.h"
8 #include "gt/intel_gt.h"
11 #include "i915_gem_clflush.h"
12 #include "i915_gem_domain.h"
13 #include "i915_gem_gtt.h"
14 #include "i915_gem_ioctls.h"
15 #include "i915_gem_lmem.h"
16 #include "i915_gem_mman.h"
17 #include "i915_gem_object.h"
20 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
22 struct drm_i915_private *i915 = to_i915(obj->base.dev);
27 return !(obj->cache_level == I915_CACHE_NONE ||
28 obj->cache_level == I915_CACHE_WT);
31 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
33 struct drm_i915_private *i915 = to_i915(obj->base.dev);
41 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
44 /* Currently in use by HW (display engine)? Keep flushed. */
45 return i915_gem_object_is_framebuffer(obj);
49 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
53 assert_object_held(obj);
55 if (!(obj->write_domain & flush_domains))
58 switch (obj->write_domain) {
59 case I915_GEM_DOMAIN_GTT:
60 spin_lock(&obj->vma.lock);
61 for_each_ggtt_vma(vma, obj) {
62 if (i915_vma_unset_ggtt_write(vma))
63 intel_gt_flush_ggtt_writes(vma->vm->gt);
65 spin_unlock(&obj->vma.lock);
67 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
70 case I915_GEM_DOMAIN_WC:
74 case I915_GEM_DOMAIN_CPU:
75 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
78 case I915_GEM_DOMAIN_RENDER:
79 if (gpu_write_needs_clflush(obj))
80 obj->cache_dirty = true;
84 obj->write_domain = 0;
87 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
90 * We manually flush the CPU domain so that we can override and
91 * force the flush for the display, and perform it asyncrhonously.
93 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
95 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
96 obj->write_domain = 0;
99 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
101 if (!i915_gem_object_is_framebuffer(obj))
104 i915_gem_object_lock(obj, NULL);
105 __i915_gem_object_flush_for_display(obj);
106 i915_gem_object_unlock(obj);
109 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
111 if (i915_gem_object_is_framebuffer(obj))
112 __i915_gem_object_flush_for_display(obj);
116 * Moves a single object to the WC read, and possibly write domain.
117 * @obj: object to act on
118 * @write: ask for write access or read only
120 * This function returns when the move is complete, including waiting on
124 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
128 assert_object_held(obj);
130 ret = i915_gem_object_wait(obj,
131 I915_WAIT_INTERRUPTIBLE |
132 (write ? I915_WAIT_ALL : 0),
133 MAX_SCHEDULE_TIMEOUT);
137 if (obj->write_domain == I915_GEM_DOMAIN_WC)
140 /* Flush and acquire obj->pages so that we are coherent through
141 * direct access in memory with previous cached writes through
142 * shmemfs and that our cache domain tracking remains valid.
143 * For example, if the obj->filp was moved to swap without us
144 * being notified and releasing the pages, we would mistakenly
145 * continue to assume that the obj remained out of the CPU cached
148 ret = i915_gem_object_pin_pages(obj);
152 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
154 /* Serialise direct access to this object with the barriers for
155 * coherent writes from the GPU, by effectively invalidating the
156 * WC domain upon first access.
158 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
161 /* It should now be out of any other write domains, and we can update
162 * the domain values for our changes.
164 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
165 obj->read_domains |= I915_GEM_DOMAIN_WC;
167 obj->read_domains = I915_GEM_DOMAIN_WC;
168 obj->write_domain = I915_GEM_DOMAIN_WC;
169 obj->mm.dirty = true;
172 i915_gem_object_unpin_pages(obj);
177 * Moves a single object to the GTT read, and possibly write domain.
178 * @obj: object to act on
179 * @write: ask for write access or read only
181 * This function returns when the move is complete, including waiting on
185 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
189 assert_object_held(obj);
191 ret = i915_gem_object_wait(obj,
192 I915_WAIT_INTERRUPTIBLE |
193 (write ? I915_WAIT_ALL : 0),
194 MAX_SCHEDULE_TIMEOUT);
198 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
201 /* Flush and acquire obj->pages so that we are coherent through
202 * direct access in memory with previous cached writes through
203 * shmemfs and that our cache domain tracking remains valid.
204 * For example, if the obj->filp was moved to swap without us
205 * being notified and releasing the pages, we would mistakenly
206 * continue to assume that the obj remained out of the CPU cached
209 ret = i915_gem_object_pin_pages(obj);
213 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
215 /* Serialise direct access to this object with the barriers for
216 * coherent writes from the GPU, by effectively invalidating the
217 * GTT domain upon first access.
219 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
222 /* It should now be out of any other write domains, and we can update
223 * the domain values for our changes.
225 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
226 obj->read_domains |= I915_GEM_DOMAIN_GTT;
228 struct i915_vma *vma;
230 obj->read_domains = I915_GEM_DOMAIN_GTT;
231 obj->write_domain = I915_GEM_DOMAIN_GTT;
232 obj->mm.dirty = true;
234 spin_lock(&obj->vma.lock);
235 for_each_ggtt_vma(vma, obj)
236 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
237 i915_vma_set_ggtt_write(vma);
238 spin_unlock(&obj->vma.lock);
241 i915_gem_object_unpin_pages(obj);
246 * Changes the cache-level of an object across all VMA.
247 * @obj: object to act on
248 * @cache_level: new cache level to set for the object
250 * After this function returns, the object will be in the new cache-level
251 * across all GTT and the contents of the backing storage will be coherent,
252 * with respect to the new cache-level. In order to keep the backing storage
253 * coherent for all users, we only allow a single cache level to be set
254 * globally on the object and prevent it from being changed whilst the
255 * hardware is reading from the object. That is if the object is currently
256 * on the scanout it will be set to uncached (or equivalent display
257 * cache coherency) and all non-MOCS GPU access will also be uncached so
258 * that all direct access to the scanout remains coherent.
260 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
261 enum i915_cache_level cache_level)
265 if (obj->cache_level == cache_level)
268 ret = i915_gem_object_wait(obj,
269 I915_WAIT_INTERRUPTIBLE |
271 MAX_SCHEDULE_TIMEOUT);
275 /* Always invalidate stale cachelines */
276 if (obj->cache_level != cache_level) {
277 i915_gem_object_set_cache_coherency(obj, cache_level);
278 obj->cache_dirty = true;
281 /* The cache-level will be applied when each vma is rebound. */
282 return i915_gem_object_unbind(obj,
283 I915_GEM_OBJECT_UNBIND_ACTIVE |
284 I915_GEM_OBJECT_UNBIND_BARRIER);
287 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
288 struct drm_file *file)
290 struct drm_i915_gem_caching *args = data;
291 struct drm_i915_gem_object *obj;
294 if (IS_DGFX(to_i915(dev)))
298 obj = i915_gem_object_lookup_rcu(file, args->handle);
304 switch (obj->cache_level) {
306 case I915_CACHE_L3_LLC:
307 args->caching = I915_CACHING_CACHED;
311 args->caching = I915_CACHING_DISPLAY;
315 args->caching = I915_CACHING_NONE;
323 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
324 struct drm_file *file)
326 struct drm_i915_private *i915 = to_i915(dev);
327 struct drm_i915_gem_caching *args = data;
328 struct drm_i915_gem_object *obj;
329 enum i915_cache_level level;
335 switch (args->caching) {
336 case I915_CACHING_NONE:
337 level = I915_CACHE_NONE;
339 case I915_CACHING_CACHED:
341 * Due to a HW issue on BXT A stepping, GPU stores via a
342 * snooped mapping may leave stale data in a corresponding CPU
343 * cacheline, whereas normally such cachelines would get
346 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
349 level = I915_CACHE_LLC;
351 case I915_CACHING_DISPLAY:
352 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
358 obj = i915_gem_object_lookup(file, args->handle);
363 * The caching mode of proxy object is handled by its generator, and
364 * not allowed to be changed by userspace.
366 if (i915_gem_object_is_proxy(obj)) {
368 * Silently allow cached for userptr; the vulkan driver
369 * sets all objects to cached
371 if (!i915_gem_object_is_userptr(obj) ||
372 args->caching != I915_CACHING_CACHED)
378 ret = i915_gem_object_lock_interruptible(obj, NULL);
382 ret = i915_gem_object_set_cache_level(obj, level);
383 i915_gem_object_unlock(obj);
386 i915_gem_object_put(obj);
391 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
392 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
393 * (for pageflips). We only flush the caches while preparing the buffer for
394 * display, the callers are responsible for frontbuffer flush.
397 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
398 struct i915_gem_ww_ctx *ww,
400 const struct i915_ggtt_view *view,
403 struct drm_i915_private *i915 = to_i915(obj->base.dev);
404 struct i915_vma *vma;
407 /* Frame buffer must be in LMEM */
408 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
409 return ERR_PTR(-EINVAL);
412 * The display engine is not coherent with the LLC cache on gen6. As
413 * a result, we make sure that the pinning that is about to occur is
414 * done with uncached PTEs. This is lowest common denominator for all
417 * However for gen6+, we could do better by using the GFDT bit instead
418 * of uncaching, which would allow us to flush all the LLC-cached data
419 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
421 ret = i915_gem_object_set_cache_level(obj,
423 I915_CACHE_WT : I915_CACHE_NONE);
428 * As the user may map the buffer once pinned in the display plane
429 * (e.g. libkms for the bootup splash), we have to ensure that we
430 * always use map_and_fenceable for all scanout buffers. However,
431 * it may simply be too big to fit into mappable, in which case
432 * put it anyway and hope that userspace can cope (but always first
433 * try to preserve the existing ABI).
435 vma = ERR_PTR(-ENOSPC);
436 if ((flags & PIN_MAPPABLE) == 0 &&
437 (!view || view->type == I915_GGTT_VIEW_NORMAL))
438 vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
439 flags | PIN_MAPPABLE |
441 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
442 vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,
447 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
448 i915_vma_mark_scanout(vma);
450 i915_gem_object_flush_if_display_locked(obj);
456 * Moves a single object to the CPU read, and possibly write domain.
457 * @obj: object to act on
458 * @write: requesting write or read-only access
460 * This function returns when the move is complete, including waiting on
464 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
468 assert_object_held(obj);
470 ret = i915_gem_object_wait(obj,
471 I915_WAIT_INTERRUPTIBLE |
472 (write ? I915_WAIT_ALL : 0),
473 MAX_SCHEDULE_TIMEOUT);
477 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
479 /* Flush the CPU cache if it's still invalid. */
480 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
481 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
482 obj->read_domains |= I915_GEM_DOMAIN_CPU;
485 /* It should now be out of any other write domains, and we can update
486 * the domain values for our changes.
488 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
490 /* If we're writing through the CPU, then the GPU read domains will
491 * need to be invalidated at next use.
494 __start_cpu_write(obj);
500 * Called when user space prepares to use an object with the CPU, either
501 * through the mmap ioctl's mapping or a GTT mapping.
503 * @data: ioctl data blob
507 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
508 struct drm_file *file)
510 struct drm_i915_gem_set_domain *args = data;
511 struct drm_i915_gem_object *obj;
512 u32 read_domains = args->read_domains;
513 u32 write_domain = args->write_domain;
516 if (IS_DGFX(to_i915(dev)))
519 /* Only handle setting domains to types used by the CPU. */
520 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
524 * Having something in the write domain implies it's in the read
525 * domain, and only that read domain. Enforce that in the request.
527 if (write_domain && read_domains != write_domain)
533 obj = i915_gem_object_lookup(file, args->handle);
538 * Try to flush the object off the GPU without holding the lock.
539 * We will repeat the flush holding the lock in the normal manner
540 * to catch cases where we are gazumped.
542 err = i915_gem_object_wait(obj,
543 I915_WAIT_INTERRUPTIBLE |
545 (write_domain ? I915_WAIT_ALL : 0),
546 MAX_SCHEDULE_TIMEOUT);
550 if (i915_gem_object_is_userptr(obj)) {
552 * Try to grab userptr pages, iris uses set_domain to check
555 err = i915_gem_object_userptr_validate(obj);
557 err = i915_gem_object_wait(obj,
558 I915_WAIT_INTERRUPTIBLE |
560 (write_domain ? I915_WAIT_ALL : 0),
561 MAX_SCHEDULE_TIMEOUT);
566 * Proxy objects do not control access to the backing storage, ergo
567 * they cannot be used as a means to manipulate the cache domain
568 * tracking for that backing storage. The proxy object is always
569 * considered to be outside of any cache domain.
571 if (i915_gem_object_is_proxy(obj)) {
576 err = i915_gem_object_lock_interruptible(obj, NULL);
581 * Flush and acquire obj->pages so that we are coherent through
582 * direct access in memory with previous cached writes through
583 * shmemfs and that our cache domain tracking remains valid.
584 * For example, if the obj->filp was moved to swap without us
585 * being notified and releasing the pages, we would mistakenly
586 * continue to assume that the obj remained out of the CPU cached
589 err = i915_gem_object_pin_pages(obj);
594 * Already in the desired write domain? Nothing for us to do!
596 * We apply a little bit of cunning here to catch a broader set of
597 * no-ops. If obj->write_domain is set, we must be in the same
598 * obj->read_domains, and only that domain. Therefore, if that
599 * obj->write_domain matches the request read_domains, we are
600 * already in the same read/write domain and can skip the operation,
601 * without having to further check the requested write_domain.
603 if (READ_ONCE(obj->write_domain) == read_domains)
606 if (read_domains & I915_GEM_DOMAIN_WC)
607 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
608 else if (read_domains & I915_GEM_DOMAIN_GTT)
609 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
611 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
614 i915_gem_object_unpin_pages(obj);
617 i915_gem_object_unlock(obj);
619 if (!err && write_domain)
620 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
623 i915_gem_object_put(obj);
628 * Pins the specified object's pages and synchronizes the object with
629 * GPU accesses. Sets needs_clflush to non-zero if the caller should
630 * flush the object from the CPU cache.
632 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
633 unsigned int *needs_clflush)
638 if (!i915_gem_object_has_struct_page(obj))
641 assert_object_held(obj);
643 ret = i915_gem_object_wait(obj,
644 I915_WAIT_INTERRUPTIBLE,
645 MAX_SCHEDULE_TIMEOUT);
649 ret = i915_gem_object_pin_pages(obj);
653 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
654 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
655 ret = i915_gem_object_set_to_cpu_domain(obj, false);
662 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
664 /* If we're not in the cpu read domain, set ourself into the gtt
665 * read domain and manually flush cachelines (if required). This
666 * optimizes for the case when the gpu will dirty the data
667 * anyway again before the next pread happens.
669 if (!obj->cache_dirty &&
670 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
671 *needs_clflush = CLFLUSH_BEFORE;
674 /* return with the pages pinned */
678 i915_gem_object_unpin_pages(obj);
682 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
683 unsigned int *needs_clflush)
688 if (!i915_gem_object_has_struct_page(obj))
691 assert_object_held(obj);
693 ret = i915_gem_object_wait(obj,
694 I915_WAIT_INTERRUPTIBLE |
696 MAX_SCHEDULE_TIMEOUT);
700 ret = i915_gem_object_pin_pages(obj);
704 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
705 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
706 ret = i915_gem_object_set_to_cpu_domain(obj, true);
713 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
715 /* If we're not in the cpu write domain, set ourself into the
716 * gtt write domain and manually flush cachelines (as required).
717 * This optimizes for the case when the gpu will use the data
718 * right away and we therefore have to clflush anyway.
720 if (!obj->cache_dirty) {
721 *needs_clflush |= CLFLUSH_AFTER;
724 * Same trick applies to invalidate partially written
725 * cachelines read before writing.
727 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
728 *needs_clflush |= CLFLUSH_BEFORE;
732 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
733 obj->mm.dirty = true;
734 /* return with the pages pinned */
738 i915_gem_object_unpin_pages(obj);