2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
24 * Daniel Vetter <daniel.vetter@ffwll.ch>
28 * DOC: frontbuffer tracking
30 * Many features require us to track changes to the currently active
31 * frontbuffer, especially rendering targeted at the frontbuffer.
33 * To be able to do so we track frontbuffers using a bitmask for all possible
34 * frontbuffer slots through intel_frontbuffer_track(). The functions in this
35 * file are then called when the contents of the frontbuffer are invalidated,
36 * when frontbuffer rendering has stopped again to flush out all the changes
37 * and when the frontbuffer is exchanged with a flip. Subsystems interested in
38 * frontbuffer changes (e.g. PSR, FBC, DRRS) should directly put their callbacks
39 * into the relevant places and filter for the frontbuffer slots that they are
42 * On a high level there are two types of powersaving features. The first one
43 * work like a special cache (FBC and PSR) and are interested when they should
44 * stop caching and when to restart caching. This is done by placing callbacks
45 * into the invalidate and the flush functions: At invalidate the caching must
46 * be stopped and at flush time it can be restarted. And maybe they need to know
47 * when the frontbuffer changes (e.g. when the hw doesn't initiate an invalidate
48 * and flush on its own) which can be achieved with placing callbacks into the
51 * The other type of display power saving feature only cares about busyness
52 * (e.g. DRRS). In that case all three (invalidate, flush and flip) indicate
53 * busyness. There is no direct way to detect idleness. Instead an idle timer
54 * work delayed work should be started from the flush and flip functions and
55 * cancelled as soon as busyness is detected.
58 #include "display/intel_dp.h"
61 #include "i915_trace.h"
62 #include "intel_display_types.h"
63 #include "intel_fbc.h"
64 #include "intel_frontbuffer.h"
65 #include "intel_drrs.h"
66 #include "intel_psr.h"
69 * frontbuffer_flush - flush frontbuffer
71 * @frontbuffer_bits: frontbuffer plane tracking bits
72 * @origin: which operation caused the flush
74 * This function gets called every time rendering on the given planes has
75 * completed and frontbuffer caching can be started again. Flushes will get
76 * delayed if they're blocked by some outstanding asynchronous rendering.
78 * Can be called without any locks held.
80 static void frontbuffer_flush(struct drm_i915_private *i915,
81 unsigned int frontbuffer_bits,
82 enum fb_op_origin origin)
84 /* Delay flushing when rings are still busy.*/
85 spin_lock(&i915->fb_tracking.lock);
86 frontbuffer_bits &= ~i915->fb_tracking.busy_bits;
87 spin_unlock(&i915->fb_tracking.lock);
89 if (!frontbuffer_bits)
92 trace_intel_frontbuffer_flush(frontbuffer_bits, origin);
95 intel_drrs_flush(i915, frontbuffer_bits);
96 intel_psr_flush(i915, frontbuffer_bits, origin);
97 intel_fbc_flush(i915, frontbuffer_bits, origin);
101 * intel_frontbuffer_flip_prepare - prepare asynchronous frontbuffer flip
103 * @frontbuffer_bits: frontbuffer plane tracking bits
105 * This function gets called after scheduling a flip on @obj. The actual
106 * frontbuffer flushing will be delayed until completion is signalled with
107 * intel_frontbuffer_flip_complete. If an invalidate happens in between this
108 * flush will be cancelled.
110 * Can be called without any locks held.
112 void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915,
113 unsigned frontbuffer_bits)
115 spin_lock(&i915->fb_tracking.lock);
116 i915->fb_tracking.flip_bits |= frontbuffer_bits;
117 /* Remove stale busy bits due to the old buffer. */
118 i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
119 spin_unlock(&i915->fb_tracking.lock);
123 * intel_frontbuffer_flip_complete - complete asynchronous frontbuffer flip
125 * @frontbuffer_bits: frontbuffer plane tracking bits
127 * This function gets called after the flip has been latched and will complete
128 * on the next vblank. It will execute the flush if it hasn't been cancelled yet.
130 * Can be called without any locks held.
132 void intel_frontbuffer_flip_complete(struct drm_i915_private *i915,
133 unsigned frontbuffer_bits)
135 spin_lock(&i915->fb_tracking.lock);
136 /* Mask any cancelled flips. */
137 frontbuffer_bits &= i915->fb_tracking.flip_bits;
138 i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
139 spin_unlock(&i915->fb_tracking.lock);
141 if (frontbuffer_bits)
142 frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
146 * intel_frontbuffer_flip - synchronous frontbuffer flip
148 * @frontbuffer_bits: frontbuffer plane tracking bits
150 * This function gets called after scheduling a flip on @obj. This is for
151 * synchronous plane updates which will happen on the next vblank and which will
152 * not get delayed by pending gpu rendering.
154 * Can be called without any locks held.
156 void intel_frontbuffer_flip(struct drm_i915_private *i915,
157 unsigned frontbuffer_bits)
159 spin_lock(&i915->fb_tracking.lock);
160 /* Remove stale busy bits due to the old buffer. */
161 i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
162 spin_unlock(&i915->fb_tracking.lock);
164 frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
167 void __intel_fb_invalidate(struct intel_frontbuffer *front,
168 enum fb_op_origin origin,
169 unsigned int frontbuffer_bits)
171 struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
173 if (origin == ORIGIN_CS) {
174 spin_lock(&i915->fb_tracking.lock);
175 i915->fb_tracking.busy_bits |= frontbuffer_bits;
176 i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
177 spin_unlock(&i915->fb_tracking.lock);
180 trace_intel_frontbuffer_invalidate(frontbuffer_bits, origin);
183 intel_psr_invalidate(i915, frontbuffer_bits, origin);
184 intel_drrs_invalidate(i915, frontbuffer_bits);
185 intel_fbc_invalidate(i915, frontbuffer_bits, origin);
188 void __intel_fb_flush(struct intel_frontbuffer *front,
189 enum fb_op_origin origin,
190 unsigned int frontbuffer_bits)
192 struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
194 if (origin == ORIGIN_CS) {
195 spin_lock(&i915->fb_tracking.lock);
196 /* Filter out new bits since rendering started. */
197 frontbuffer_bits &= i915->fb_tracking.busy_bits;
198 i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
199 spin_unlock(&i915->fb_tracking.lock);
202 if (frontbuffer_bits)
203 frontbuffer_flush(i915, frontbuffer_bits, origin);
206 static int frontbuffer_active(struct i915_active *ref)
208 struct intel_frontbuffer *front =
209 container_of(ref, typeof(*front), write);
211 kref_get(&front->ref);
215 static void frontbuffer_retire(struct i915_active *ref)
217 struct intel_frontbuffer *front =
218 container_of(ref, typeof(*front), write);
220 intel_frontbuffer_flush(front, ORIGIN_CS);
221 intel_frontbuffer_put(front);
224 static void frontbuffer_release(struct kref *ref)
225 __releases(&to_i915(front->obj->base.dev)->fb_tracking.lock)
227 struct intel_frontbuffer *front =
228 container_of(ref, typeof(*front), ref);
229 struct drm_i915_gem_object *obj = front->obj;
230 struct i915_vma *vma;
232 drm_WARN_ON(obj->base.dev, atomic_read(&front->bits));
234 spin_lock(&obj->vma.lock);
235 for_each_ggtt_vma(vma, obj) {
236 i915_vma_clear_scanout(vma);
237 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
239 spin_unlock(&obj->vma.lock);
241 RCU_INIT_POINTER(obj->frontbuffer, NULL);
242 spin_unlock(&to_i915(obj->base.dev)->fb_tracking.lock);
244 i915_active_fini(&front->write);
246 i915_gem_object_put(obj);
247 kfree_rcu(front, rcu);
250 struct intel_frontbuffer *
251 intel_frontbuffer_get(struct drm_i915_gem_object *obj)
253 struct drm_i915_private *i915 = to_i915(obj->base.dev);
254 struct intel_frontbuffer *front;
256 front = __intel_frontbuffer_get(obj);
260 front = kmalloc(sizeof(*front), GFP_KERNEL);
265 kref_init(&front->ref);
266 atomic_set(&front->bits, 0);
267 i915_active_init(&front->write,
270 I915_ACTIVE_RETIRE_SLEEPS);
272 spin_lock(&i915->fb_tracking.lock);
273 if (rcu_access_pointer(obj->frontbuffer)) {
275 front = rcu_dereference_protected(obj->frontbuffer, true);
276 kref_get(&front->ref);
278 i915_gem_object_get(obj);
279 rcu_assign_pointer(obj->frontbuffer, front);
281 spin_unlock(&i915->fb_tracking.lock);
286 void intel_frontbuffer_put(struct intel_frontbuffer *front)
288 kref_put_lock(&front->ref,
290 &to_i915(front->obj->base.dev)->fb_tracking.lock);
294 * intel_frontbuffer_track - update frontbuffer tracking
295 * @old: current buffer for the frontbuffer slots
296 * @new: new buffer for the frontbuffer slots
297 * @frontbuffer_bits: bitmask of frontbuffer slots
299 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
300 * from @old and setting them in @new. Both @old and @new can be NULL.
302 void intel_frontbuffer_track(struct intel_frontbuffer *old,
303 struct intel_frontbuffer *new,
304 unsigned int frontbuffer_bits)
307 * Control of individual bits within the mask are guarded by
308 * the owning plane->mutex, i.e. we can never see concurrent
309 * manipulation of individual bits. But since the bitfield as a whole
310 * is updated using RMW, we need to use atomics in order to update
313 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
314 BITS_PER_TYPE(atomic_t));
317 drm_WARN_ON(old->obj->base.dev,
318 !(atomic_read(&old->bits) & frontbuffer_bits));
319 atomic_andnot(frontbuffer_bits, &old->bits);
323 drm_WARN_ON(new->obj->base.dev,
324 atomic_read(&new->bits) & frontbuffer_bits);
325 atomic_or(frontbuffer_bits, &new->bits);