2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/kthread.h>
26 #include <trace/events/dma_fence.h>
27 #include <uapi/linux/sched/types.h>
31 static void irq_enable(struct intel_engine_cs *engine)
33 if (!engine->irq_enable)
36 /* Caller disables interrupts */
37 spin_lock(&engine->i915->irq_lock);
38 engine->irq_enable(engine);
39 spin_unlock(&engine->i915->irq_lock);
42 static void irq_disable(struct intel_engine_cs *engine)
44 if (!engine->irq_disable)
47 /* Caller disables interrupts */
48 spin_lock(&engine->i915->irq_lock);
49 engine->irq_disable(engine);
50 spin_unlock(&engine->i915->irq_lock);
53 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
55 lockdep_assert_held(&b->irq_lock);
57 GEM_BUG_ON(!b->irq_enabled);
58 if (!--b->irq_enabled)
59 irq_disable(container_of(b,
60 struct intel_engine_cs,
66 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
68 struct intel_breadcrumbs *b = &engine->breadcrumbs;
73 spin_lock_irq(&b->irq_lock);
75 __intel_breadcrumbs_disarm_irq(b);
76 spin_unlock_irq(&b->irq_lock);
79 static inline bool __request_completed(const struct i915_request *rq)
81 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
85 __dma_fence_signal(struct dma_fence *fence)
87 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
91 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
93 fence->timestamp = timestamp;
94 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
95 trace_dma_fence_signaled(fence);
99 __dma_fence_signal__notify(struct dma_fence *fence)
101 struct dma_fence_cb *cur, *tmp;
103 lockdep_assert_held(fence->lock);
104 lockdep_assert_irqs_disabled();
106 list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
107 INIT_LIST_HEAD(&cur->node);
108 cur->func(fence, cur);
110 INIT_LIST_HEAD(&fence->cb_list);
113 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
115 struct intel_breadcrumbs *b = &engine->breadcrumbs;
116 const ktime_t timestamp = ktime_get();
117 struct intel_context *ce, *cn;
118 struct list_head *pos, *next;
121 spin_lock(&b->irq_lock);
123 if (b->irq_armed && list_empty(&b->signalers))
124 __intel_breadcrumbs_disarm_irq(b);
126 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
127 GEM_BUG_ON(list_empty(&ce->signals));
129 list_for_each_safe(pos, next, &ce->signals) {
130 struct i915_request *rq =
131 list_entry(pos, typeof(*rq), signal_link);
133 if (!__request_completed(rq))
136 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
138 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
140 if (!__dma_fence_signal(&rq->fence))
144 * Queue for execution after dropping the signaling
145 * spinlock as the callback chain may end up adding
146 * more signalers to the same context or engine.
148 i915_request_get(rq);
149 list_add_tail(&rq->signal_link, &signal);
153 * We process the list deletion in bulk, only using a list_add
154 * (not list_move) above but keeping the status of
155 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
157 if (!list_is_first(pos, &ce->signals)) {
158 /* Advance the list to the first incomplete request */
159 __list_del_many(&ce->signals, pos);
160 if (&ce->signals == pos) /* now empty */
161 list_del_init(&ce->signal_link);
165 spin_unlock(&b->irq_lock);
167 list_for_each_safe(pos, next, &signal) {
168 struct i915_request *rq =
169 list_entry(pos, typeof(*rq), signal_link);
171 __dma_fence_signal__timestamp(&rq->fence, timestamp);
173 spin_lock(&rq->lock);
174 __dma_fence_signal__notify(&rq->fence);
175 spin_unlock(&rq->lock);
177 i915_request_put(rq);
181 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
184 intel_engine_breadcrumbs_irq(engine);
188 static void signal_irq_work(struct irq_work *work)
190 struct intel_engine_cs *engine =
191 container_of(work, typeof(*engine), breadcrumbs.irq_work);
193 intel_engine_breadcrumbs_irq(engine);
196 void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine)
198 struct intel_breadcrumbs *b = &engine->breadcrumbs;
200 spin_lock_irq(&b->irq_lock);
201 if (!b->irq_enabled++)
203 GEM_BUG_ON(!b->irq_enabled); /* no overflow! */
204 spin_unlock_irq(&b->irq_lock);
207 void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine)
209 struct intel_breadcrumbs *b = &engine->breadcrumbs;
211 spin_lock_irq(&b->irq_lock);
212 GEM_BUG_ON(!b->irq_enabled); /* no underflow! */
213 if (!--b->irq_enabled)
215 spin_unlock_irq(&b->irq_lock);
218 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
220 struct intel_engine_cs *engine =
221 container_of(b, struct intel_engine_cs, breadcrumbs);
223 lockdep_assert_held(&b->irq_lock);
228 * The breadcrumb irq will be disarmed on the interrupt after the
229 * waiters are signaled. This gives us a single interrupt window in
230 * which we can add a new waiter and avoid the cost of re-enabling
236 * Since we are waiting on a request, the GPU should be busy
237 * and should have its own rpm reference. This is tracked
238 * by i915->gt.awake, we can forgo holding our own wakref
239 * for the interrupt as before i915->gt.awake is released (when
240 * the driver is idle) we disarm the breadcrumbs.
243 if (!b->irq_enabled++)
247 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
249 struct intel_breadcrumbs *b = &engine->breadcrumbs;
251 spin_lock_init(&b->irq_lock);
252 INIT_LIST_HEAD(&b->signalers);
254 init_irq_work(&b->irq_work, signal_irq_work);
257 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
259 struct intel_breadcrumbs *b = &engine->breadcrumbs;
262 spin_lock_irqsave(&b->irq_lock, flags);
269 spin_unlock_irqrestore(&b->irq_lock, flags);
272 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
276 bool i915_request_enable_breadcrumb(struct i915_request *rq)
278 lockdep_assert_held(&rq->lock);
279 lockdep_assert_irqs_disabled();
281 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
282 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
283 struct intel_context *ce = rq->hw_context;
284 struct list_head *pos;
286 spin_lock(&b->irq_lock);
287 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
289 __intel_breadcrumbs_arm_irq(b);
292 * We keep the seqno in retirement order, so we can break
293 * inside intel_engine_breadcrumbs_irq as soon as we've passed
294 * the last completed request (or seen a request that hasn't
295 * event started). We could iterate the timeline->requests list,
296 * but keeping a separate signalers_list has the advantage of
297 * hopefully being much smaller than the full list and so
298 * provides faster iteration and detection when there are no
299 * more interrupts required for this context.
301 * We typically expect to add new signalers in order, so we
302 * start looking for our insertion point from the tail of
305 list_for_each_prev(pos, &ce->signals) {
306 struct i915_request *it =
307 list_entry(pos, typeof(*it), signal_link);
309 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
312 list_add(&rq->signal_link, pos);
313 if (pos == &ce->signals) /* catch transitions from empty list */
314 list_move_tail(&ce->signal_link, &b->signalers);
316 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
317 spin_unlock(&b->irq_lock);
320 return !__request_completed(rq);
323 void i915_request_cancel_breadcrumb(struct i915_request *rq)
325 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
327 lockdep_assert_held(&rq->lock);
328 lockdep_assert_irqs_disabled();
331 * We must wait for b->irq_lock so that we know the interrupt handler
332 * has released its reference to the intel_context and has completed
333 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
336 spin_lock(&b->irq_lock);
337 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
338 struct intel_context *ce = rq->hw_context;
340 list_del(&rq->signal_link);
341 if (list_empty(&ce->signals))
342 list_del_init(&ce->signal_link);
344 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
346 spin_unlock(&b->irq_lock);
349 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
350 struct drm_printer *p)
352 struct intel_breadcrumbs *b = &engine->breadcrumbs;
353 struct intel_context *ce;
354 struct i915_request *rq;
356 if (list_empty(&b->signalers))
359 drm_printf(p, "Signals:\n");
361 spin_lock_irq(&b->irq_lock);
362 list_for_each_entry(ce, &b->signalers, signal_link) {
363 list_for_each_entry(rq, &ce->signals, signal_link) {
364 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
365 rq->fence.context, rq->fence.seqno,
366 i915_request_completed(rq) ? "!" :
367 i915_request_started(rq) ? "*" :
369 jiffies_to_msecs(jiffies - rq->emitted_jiffies));
372 spin_unlock_irq(&b->irq_lock);