1 // SPDX-License-Identifier: MIT
3 * Copyright © 2019 Intel Corporation
7 #include "i915_request.h"
9 #include "intel_context.h"
10 #include "intel_engine_heartbeat.h"
11 #include "intel_engine_pm.h"
12 #include "intel_engine.h"
14 #include "intel_reset.h"
17 * While the engine is active, we send a periodic pulse along the engine
18 * to check on its health and to flush any idle-barriers. If that request
19 * is stuck, and we fail to preempt it, we declare the engine hung and
20 * issue a reset -- in the hope that restores progress.
23 static bool next_heartbeat(struct intel_engine_cs *engine)
27 delay = READ_ONCE(engine->props.heartbeat_interval_ms);
31 delay = msecs_to_jiffies_timeout(delay);
33 delay = round_jiffies_up_relative(delay);
34 mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
39 static struct i915_request *
40 heartbeat_create(struct intel_context *ce, gfp_t gfp)
42 struct i915_request *rq;
44 intel_context_enter(ce);
45 rq = __i915_request_create(ce, gfp);
46 intel_context_exit(ce);
51 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
53 engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
54 i915_request_add_active_barriers(rq);
55 if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
56 engine->heartbeat.systole = i915_request_get(rq);
59 static void heartbeat_commit(struct i915_request *rq,
60 const struct i915_sched_attr *attr)
62 idle_pulse(rq->engine, rq);
64 __i915_request_commit(rq);
65 __i915_request_queue(rq, attr);
68 static void show_heartbeat(const struct i915_request *rq,
69 struct intel_engine_cs *engine)
71 struct drm_printer p = drm_debug_printer("heartbeat");
73 intel_engine_dump(engine, &p,
74 "%s heartbeat {seqno:%llx:%lld, prio:%d} not ticking\n",
78 rq->sched.attr.priority);
81 static void heartbeat(struct work_struct *wrk)
83 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
84 struct intel_engine_cs *engine =
85 container_of(wrk, typeof(*engine), heartbeat.work.work);
86 struct intel_context *ce = engine->kernel_context;
87 struct i915_request *rq;
90 /* Just in case everything has gone horribly wrong, give it a kick */
91 intel_engine_flush_submission(engine);
93 rq = engine->heartbeat.systole;
94 if (rq && i915_request_completed(rq)) {
96 engine->heartbeat.systole = NULL;
99 if (!intel_engine_pm_get_if_awake(engine))
102 if (intel_gt_is_wedged(engine->gt))
105 if (engine->heartbeat.systole) {
106 long delay = READ_ONCE(engine->props.heartbeat_interval_ms);
108 /* Safeguard against too-fast worker invocations */
109 if (!time_after(jiffies,
110 rq->emitted_jiffies + msecs_to_jiffies(delay)))
113 if (!i915_sw_fence_signaled(&rq->submit)) {
115 * Not yet submitted, system is stalled.
117 * This more often happens for ring submission,
118 * where all contexts are funnelled into a common
119 * ringbuffer. If one context is blocked on an
120 * external fence, not only is it not submitted,
121 * but all other contexts, including the kernel
122 * context are stuck waiting for the signal.
124 } else if (engine->schedule &&
125 rq->sched.attr.priority < I915_PRIORITY_BARRIER) {
127 * Gradually raise the priority of the heartbeat to
128 * give high priority work [which presumably desires
129 * low latency and no jitter] the chance to naturally
130 * complete before being preempted.
133 if (rq->sched.attr.priority >= attr.priority)
134 attr.priority = I915_PRIORITY_HEARTBEAT;
135 if (rq->sched.attr.priority >= attr.priority)
136 attr.priority = I915_PRIORITY_BARRIER;
139 engine->schedule(rq, &attr);
142 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
143 show_heartbeat(rq, engine);
145 intel_gt_handle_error(engine->gt, engine->mask,
147 "stopped heartbeat on %s",
151 rq->emitted_jiffies = jiffies;
155 serial = READ_ONCE(engine->serial);
156 if (engine->wakeref_serial == serial)
159 if (!mutex_trylock(&ce->timeline->mutex)) {
160 /* Unable to lock the kernel timeline, is the engine stuck? */
161 if (xchg(&engine->heartbeat.blocked, serial) == serial)
162 intel_gt_handle_error(engine->gt, engine->mask,
164 "no heartbeat on %s",
169 rq = heartbeat_create(ce, GFP_NOWAIT | __GFP_NOWARN);
173 heartbeat_commit(rq, &attr);
176 mutex_unlock(&ce->timeline->mutex);
178 if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
179 i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
180 intel_engine_pm_put(engine);
183 void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
185 if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL))
188 next_heartbeat(engine);
191 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
193 if (cancel_delayed_work(&engine->heartbeat.work))
194 i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
197 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
199 INIT_DELAYED_WORK(&engine->heartbeat.work, heartbeat);
202 static int __intel_engine_pulse(struct intel_engine_cs *engine)
204 struct i915_sched_attr attr = { .priority = I915_PRIORITY_BARRIER };
205 struct intel_context *ce = engine->kernel_context;
206 struct i915_request *rq;
208 lockdep_assert_held(&ce->timeline->mutex);
209 GEM_BUG_ON(!intel_engine_has_preemption(engine));
210 GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
212 rq = heartbeat_create(ce, GFP_NOWAIT | __GFP_NOWARN);
216 __set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
218 heartbeat_commit(rq, &attr);
219 GEM_BUG_ON(rq->sched.attr.priority < I915_PRIORITY_BARRIER);
224 static unsigned long set_heartbeat(struct intel_engine_cs *engine,
229 old = xchg(&engine->props.heartbeat_interval_ms, delay);
231 intel_engine_unpark_heartbeat(engine);
233 intel_engine_park_heartbeat(engine);
238 int intel_engine_set_heartbeat(struct intel_engine_cs *engine,
241 struct intel_context *ce = engine->kernel_context;
244 if (!delay && !intel_engine_has_preempt_reset(engine))
247 intel_engine_pm_get(engine);
249 err = mutex_lock_interruptible(&ce->timeline->mutex);
253 if (delay != engine->props.heartbeat_interval_ms) {
254 unsigned long saved = set_heartbeat(engine, delay);
256 /* recheck current execution */
257 if (intel_engine_has_preemption(engine)) {
258 err = __intel_engine_pulse(engine);
260 set_heartbeat(engine, saved);
264 mutex_unlock(&ce->timeline->mutex);
267 intel_engine_pm_put(engine);
271 int intel_engine_pulse(struct intel_engine_cs *engine)
273 struct intel_context *ce = engine->kernel_context;
276 if (!intel_engine_has_preemption(engine))
279 if (!intel_engine_pm_get_if_awake(engine))
283 if (!mutex_lock_interruptible(&ce->timeline->mutex)) {
284 err = __intel_engine_pulse(engine);
285 mutex_unlock(&ce->timeline->mutex);
288 intel_engine_flush_submission(engine);
289 intel_engine_pm_put(engine);
293 int intel_engine_flush_barriers(struct intel_engine_cs *engine)
295 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
296 struct intel_context *ce = engine->kernel_context;
297 struct i915_request *rq;
300 if (llist_empty(&engine->barrier_tasks))
303 if (!intel_engine_pm_get_if_awake(engine))
306 if (mutex_lock_interruptible(&ce->timeline->mutex)) {
311 rq = heartbeat_create(ce, GFP_KERNEL);
317 heartbeat_commit(rq, &attr);
321 mutex_unlock(&ce->timeline->mutex);
323 intel_engine_pm_put(engine);
327 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
328 #include "selftest_engine_heartbeat.c"