1 // SPDX-License-Identifier: MIT
3 * Copyright © 2018 Intel Corporation
6 #include <linux/prime_numbers.h>
8 #include "gem/i915_gem_internal.h"
9 #include "gem/i915_gem_pm.h"
10 #include "gt/intel_engine_heartbeat.h"
11 #include "gt/intel_reset.h"
12 #include "gt/selftest_engine_heartbeat.h"
14 #include "i915_selftest.h"
15 #include "selftests/i915_random.h"
16 #include "selftests/igt_flush_test.h"
17 #include "selftests/igt_live_test.h"
18 #include "selftests/igt_spinner.h"
19 #include "selftests/lib_sw_fence.h"
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
28 static bool is_active(struct i915_request *rq)
30 if (i915_request_is_active(rq))
33 if (i915_request_on_hold(rq))
36 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
42 static int wait_for_submit(struct intel_engine_cs *engine,
43 struct i915_request *rq,
44 unsigned long timeout)
46 /* Ignore our own attempts to suppress excess tasklets */
47 tasklet_hi_schedule(&engine->sched_engine->tasklet);
51 bool done = time_after(jiffies, timeout);
53 if (i915_request_completed(rq)) /* that was quick! */
56 /* Wait until the HW has acknowleged the submission (or err) */
57 intel_engine_flush_submission(engine);
58 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
68 static int wait_for_reset(struct intel_engine_cs *engine,
69 struct i915_request *rq,
70 unsigned long timeout)
76 intel_engine_flush_submission(engine);
78 if (READ_ONCE(engine->execlists.pending[0]))
81 if (i915_request_completed(rq))
84 if (READ_ONCE(rq->fence.error))
86 } while (time_before(jiffies, timeout));
88 flush_scheduled_work();
90 if (rq->fence.error != -EIO) {
91 pr_err("%s: hanging request %llx:%lld not reset\n",
98 /* Give the request a jiffie to complete after flushing the worker */
99 if (i915_request_wait(rq, 0,
100 max(0l, (long)(timeout - jiffies)) + 1) < 0) {
101 pr_err("%s: hanging request %llx:%lld did not complete\n",
111 static int live_sanitycheck(void *arg)
113 struct intel_gt *gt = arg;
114 struct intel_engine_cs *engine;
115 enum intel_engine_id id;
116 struct igt_spinner spin;
119 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
122 if (igt_spinner_init(&spin, gt))
125 for_each_engine(engine, gt, id) {
126 struct intel_context *ce;
127 struct i915_request *rq;
129 ce = intel_context_create(engine);
135 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
141 i915_request_add(rq);
142 if (!igt_wait_for_spinner(&spin, rq)) {
143 GEM_TRACE("spinner failed to start\n");
145 intel_gt_set_wedged(gt);
150 igt_spinner_end(&spin);
151 if (igt_flush_test(gt->i915)) {
157 intel_context_put(ce);
162 igt_spinner_fini(&spin);
166 static int live_unlite_restore(struct intel_gt *gt, int prio)
168 struct intel_engine_cs *engine;
169 enum intel_engine_id id;
170 struct igt_spinner spin;
174 * Check that we can correctly context switch between 2 instances
175 * on the same engine from the same parent context.
178 if (igt_spinner_init(&spin, gt))
182 for_each_engine(engine, gt, id) {
183 struct intel_context *ce[2] = {};
184 struct i915_request *rq[2];
185 struct igt_live_test t;
188 if (prio && !intel_engine_has_preemption(engine))
191 if (!intel_engine_can_store_dword(engine))
194 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
198 st_engine_heartbeat_disable(engine);
200 for (n = 0; n < ARRAY_SIZE(ce); n++) {
201 struct intel_context *tmp;
203 tmp = intel_context_create(engine);
209 err = intel_context_pin(tmp);
211 intel_context_put(tmp);
216 * Setup the pair of contexts such that if we
217 * lite-restore using the RING_TAIL from ce[1] it
218 * will execute garbage from ce[0]->ring.
220 memset(tmp->ring->vaddr,
221 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
222 tmp->ring->vma->size);
226 GEM_BUG_ON(!ce[1]->ring->size);
227 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
228 lrc_update_regs(ce[1], engine, ce[1]->ring->head);
230 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
232 err = PTR_ERR(rq[0]);
236 i915_request_get(rq[0]);
237 i915_request_add(rq[0]);
238 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
240 if (!igt_wait_for_spinner(&spin, rq[0])) {
241 i915_request_put(rq[0]);
245 rq[1] = i915_request_create(ce[1]);
247 err = PTR_ERR(rq[1]);
248 i915_request_put(rq[0]);
254 * Ensure we do the switch to ce[1] on completion.
256 * rq[0] is already submitted, so this should reduce
257 * to a no-op (a wait on a request on the same engine
258 * uses the submit fence, not the completion fence),
259 * but it will install a dependency on rq[1] for rq[0]
260 * that will prevent the pair being reordered by
263 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
266 i915_request_get(rq[1]);
267 i915_request_add(rq[1]);
268 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
269 i915_request_put(rq[0]);
272 struct i915_sched_attr attr = {
276 /* Alternatively preempt the spinner with ce[1] */
277 engine->sched_engine->schedule(rq[1], &attr);
280 /* And switch back to ce[0] for good measure */
281 rq[0] = i915_request_create(ce[0]);
283 err = PTR_ERR(rq[0]);
284 i915_request_put(rq[1]);
288 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
289 i915_request_get(rq[0]);
290 i915_request_add(rq[0]);
291 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
292 i915_request_put(rq[1]);
293 i915_request_put(rq[0]);
296 intel_engine_flush_submission(engine);
297 igt_spinner_end(&spin);
298 for (n = 0; n < ARRAY_SIZE(ce); n++) {
299 if (IS_ERR_OR_NULL(ce[n]))
302 intel_context_unpin(ce[n]);
303 intel_context_put(ce[n]);
306 st_engine_heartbeat_enable(engine);
307 if (igt_live_test_end(&t))
313 igt_spinner_fini(&spin);
317 static int live_unlite_switch(void *arg)
319 return live_unlite_restore(arg, 0);
322 static int live_unlite_preempt(void *arg)
324 return live_unlite_restore(arg, I915_PRIORITY_MAX);
327 static int live_unlite_ring(void *arg)
329 struct intel_gt *gt = arg;
330 struct intel_engine_cs *engine;
331 struct igt_spinner spin;
332 enum intel_engine_id id;
336 * Setup a preemption event that will cause almost the entire ring
337 * to be unwound, potentially fooling our intel_ring_direction()
338 * into emitting a forward lite-restore instead of the rollback.
341 if (igt_spinner_init(&spin, gt))
344 for_each_engine(engine, gt, id) {
345 struct intel_context *ce[2] = {};
346 struct i915_request *rq;
347 struct igt_live_test t;
350 if (!intel_engine_has_preemption(engine))
353 if (!intel_engine_can_store_dword(engine))
356 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
360 st_engine_heartbeat_disable(engine);
362 for (n = 0; n < ARRAY_SIZE(ce); n++) {
363 struct intel_context *tmp;
365 tmp = intel_context_create(engine);
371 err = intel_context_pin(tmp);
373 intel_context_put(tmp);
377 memset32(tmp->ring->vaddr,
378 0xdeadbeef, /* trigger a hang if executed */
379 tmp->ring->vma->size / sizeof(u32));
384 /* Create max prio spinner, followed by N low prio nops */
385 rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
391 i915_request_get(rq);
392 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
393 i915_request_add(rq);
395 if (!igt_wait_for_spinner(&spin, rq)) {
396 intel_gt_set_wedged(gt);
397 i915_request_put(rq);
402 /* Fill the ring, until we will cause a wrap */
404 while (intel_ring_direction(ce[0]->ring,
406 ce[0]->ring->tail) <= 0) {
407 struct i915_request *tmp;
409 tmp = intel_context_create_request(ce[0]);
412 i915_request_put(rq);
416 i915_request_add(tmp);
417 intel_engine_flush_submission(engine);
420 intel_engine_flush_submission(engine);
421 pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
427 GEM_BUG_ON(intel_ring_direction(ce[0]->ring,
429 ce[0]->ring->tail) <= 0);
430 i915_request_put(rq);
432 /* Create a second ring to preempt the first ring after rq[0] */
433 rq = intel_context_create_request(ce[1]);
439 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
440 i915_request_get(rq);
441 i915_request_add(rq);
443 err = wait_for_submit(engine, rq, HZ / 2);
444 i915_request_put(rq);
446 pr_err("%s: preemption request was not submitted\n",
451 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
453 ce[0]->ring->tail, ce[0]->ring->emit,
454 ce[1]->ring->tail, ce[1]->ring->emit);
457 intel_engine_flush_submission(engine);
458 igt_spinner_end(&spin);
459 for (n = 0; n < ARRAY_SIZE(ce); n++) {
460 if (IS_ERR_OR_NULL(ce[n]))
463 intel_context_unpin(ce[n]);
464 intel_context_put(ce[n]);
466 st_engine_heartbeat_enable(engine);
467 if (igt_live_test_end(&t))
473 igt_spinner_fini(&spin);
477 static int live_pin_rewind(void *arg)
479 struct intel_gt *gt = arg;
480 struct intel_engine_cs *engine;
481 enum intel_engine_id id;
485 * We have to be careful not to trust intel_ring too much, for example
486 * ring->head is updated upon retire which is out of sync with pinning
487 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
488 * or else we risk writing an older, stale value.
490 * To simulate this, let's apply a bit of deliberate sabotague.
493 for_each_engine(engine, gt, id) {
494 struct intel_context *ce;
495 struct i915_request *rq;
496 struct intel_ring *ring;
497 struct igt_live_test t;
499 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
504 ce = intel_context_create(engine);
510 err = intel_context_pin(ce);
512 intel_context_put(ce);
516 /* Keep the context awake while we play games */
517 err = i915_active_acquire(&ce->active);
519 intel_context_unpin(ce);
520 intel_context_put(ce);
525 /* Poison the ring, and offset the next request from HEAD */
526 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
527 ring->emit = ring->size / 2;
528 ring->tail = ring->emit;
529 GEM_BUG_ON(ring->head);
531 intel_context_unpin(ce);
533 /* Submit a simple nop request */
534 GEM_BUG_ON(intel_context_is_pinned(ce));
535 rq = intel_context_create_request(ce);
536 i915_active_release(&ce->active); /* e.g. async retire */
537 intel_context_put(ce);
542 GEM_BUG_ON(!rq->head);
543 i915_request_add(rq);
545 /* Expect not to hang! */
546 if (igt_live_test_end(&t)) {
555 static int engine_lock_reset_tasklet(struct intel_engine_cs *engine)
557 tasklet_disable(&engine->sched_engine->tasklet);
560 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
561 &engine->gt->reset.flags)) {
563 tasklet_enable(&engine->sched_engine->tasklet);
565 intel_gt_set_wedged(engine->gt);
572 static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine)
574 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
575 &engine->gt->reset.flags);
578 tasklet_enable(&engine->sched_engine->tasklet);
581 static int live_hold_reset(void *arg)
583 struct intel_gt *gt = arg;
584 struct intel_engine_cs *engine;
585 enum intel_engine_id id;
586 struct igt_spinner spin;
590 * In order to support offline error capture for fast preempt reset,
591 * we need to decouple the guilty request and ensure that it and its
592 * descendents are not executed while the capture is in progress.
595 if (!intel_has_reset_engine(gt))
598 if (igt_spinner_init(&spin, gt))
601 for_each_engine(engine, gt, id) {
602 struct intel_context *ce;
603 struct i915_request *rq;
605 ce = intel_context_create(engine);
611 st_engine_heartbeat_disable(engine);
613 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
618 i915_request_add(rq);
620 if (!igt_wait_for_spinner(&spin, rq)) {
621 intel_gt_set_wedged(gt);
626 /* We have our request executing, now remove it and reset */
628 err = engine_lock_reset_tasklet(engine);
632 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
633 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
635 i915_request_get(rq);
636 execlists_hold(engine, rq);
637 GEM_BUG_ON(!i915_request_on_hold(rq));
639 __intel_engine_reset_bh(engine, NULL);
640 GEM_BUG_ON(rq->fence.error != -EIO);
642 engine_unlock_reset_tasklet(engine);
644 /* Check that we do not resubmit the held request */
645 if (!i915_request_wait(rq, 0, HZ / 5)) {
646 pr_err("%s: on hold request completed!\n",
648 i915_request_put(rq);
652 GEM_BUG_ON(!i915_request_on_hold(rq));
654 /* But is resubmitted on release */
655 execlists_unhold(engine, rq);
656 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
657 pr_err("%s: held request did not complete!\n",
659 intel_gt_set_wedged(gt);
662 i915_request_put(rq);
665 st_engine_heartbeat_enable(engine);
666 intel_context_put(ce);
671 igt_spinner_fini(&spin);
675 static const char *error_repr(int err)
677 return err ? "bad" : "good";
680 static int live_error_interrupt(void *arg)
682 static const struct error_phase {
683 enum { GOOD = 0, BAD = -EIO } error[2];
688 { { GOOD, GOOD } }, /* sentinel */
690 struct intel_gt *gt = arg;
691 struct intel_engine_cs *engine;
692 enum intel_engine_id id;
695 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
696 * of invalid commands in user batches that will cause a GPU hang.
697 * This is a faster mechanism than using hangcheck/heartbeats, but
698 * only detects problems the HW knows about -- it will not warn when
701 * To verify our detection and reset, we throw some invalid commands
702 * at the HW and wait for the interrupt.
705 if (!intel_has_reset_engine(gt))
708 for_each_engine(engine, gt, id) {
709 const struct error_phase *p;
712 st_engine_heartbeat_disable(engine);
714 for (p = phases; p->error[0] != GOOD; p++) {
715 struct i915_request *client[ARRAY_SIZE(phases->error)];
719 memset(client, 0, sizeof(*client));
720 for (i = 0; i < ARRAY_SIZE(client); i++) {
721 struct intel_context *ce;
722 struct i915_request *rq;
724 ce = intel_context_create(engine);
730 rq = intel_context_create_request(ce);
731 intel_context_put(ce);
737 if (rq->engine->emit_init_breadcrumb) {
738 err = rq->engine->emit_init_breadcrumb(rq);
740 i915_request_add(rq);
745 cs = intel_ring_begin(rq, 2);
747 i915_request_add(rq);
760 client[i] = i915_request_get(rq);
761 i915_request_add(rq);
764 err = wait_for_submit(engine, client[0], HZ / 2);
766 pr_err("%s: first request did not start within time!\n",
772 for (i = 0; i < ARRAY_SIZE(client); i++) {
773 if (i915_request_wait(client[i], 0, HZ / 5) < 0)
774 pr_debug("%s: %s request incomplete!\n",
776 error_repr(p->error[i]));
778 if (!i915_request_started(client[i])) {
779 pr_err("%s: %s request not started!\n",
781 error_repr(p->error[i]));
786 /* Kick the tasklet to process the error */
787 intel_engine_flush_submission(engine);
788 if (client[i]->fence.error != p->error[i]) {
789 pr_err("%s: %s request (%s) with wrong error code: %d\n",
791 error_repr(p->error[i]),
792 i915_request_completed(client[i]) ? "completed" : "running",
793 client[i]->fence.error);
800 for (i = 0; i < ARRAY_SIZE(client); i++)
802 i915_request_put(client[i]);
804 pr_err("%s: failed at phase[%zd] { %d, %d }\n",
805 engine->name, p - phases,
806 p->error[0], p->error[1]);
811 st_engine_heartbeat_enable(engine);
813 intel_gt_set_wedged(gt);
822 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
826 cs = intel_ring_begin(rq, 10);
830 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
832 *cs++ = MI_SEMAPHORE_WAIT |
833 MI_SEMAPHORE_GLOBAL_GTT |
835 MI_SEMAPHORE_SAD_NEQ_SDD;
837 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
841 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
842 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
852 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
854 intel_ring_advance(rq, cs);
858 static struct i915_request *
859 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
861 struct intel_context *ce;
862 struct i915_request *rq;
865 ce = intel_context_create(engine);
869 rq = intel_context_create_request(ce);
874 if (rq->engine->emit_init_breadcrumb)
875 err = rq->engine->emit_init_breadcrumb(rq);
877 err = emit_semaphore_chain(rq, vma, idx);
879 i915_request_get(rq);
880 i915_request_add(rq);
885 intel_context_put(ce);
890 release_queue(struct intel_engine_cs *engine,
891 struct i915_vma *vma,
894 struct i915_sched_attr attr = {
897 struct i915_request *rq;
900 rq = intel_engine_create_kernel_request(engine);
904 cs = intel_ring_begin(rq, 4);
906 i915_request_add(rq);
910 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
911 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
915 intel_ring_advance(rq, cs);
917 i915_request_get(rq);
918 i915_request_add(rq);
921 engine->sched_engine->schedule(rq, &attr);
922 local_bh_enable(); /* kick tasklet */
924 i915_request_put(rq);
930 slice_semaphore_queue(struct intel_engine_cs *outer,
931 struct i915_vma *vma,
934 struct intel_engine_cs *engine;
935 struct i915_request *head;
936 enum intel_engine_id id;
939 head = semaphore_queue(outer, vma, n++);
941 return PTR_ERR(head);
943 for_each_engine(engine, outer->gt, id) {
944 if (!intel_engine_has_preemption(engine))
947 for (i = 0; i < count; i++) {
948 struct i915_request *rq;
950 rq = semaphore_queue(engine, vma, n++);
956 i915_request_put(rq);
960 err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
964 if (i915_request_wait(head, 0,
965 2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {
966 pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n",
967 outer->name, count, n);
969 intel_gt_set_wedged(outer->gt);
974 i915_request_put(head);
978 static int live_timeslice_preempt(void *arg)
980 struct intel_gt *gt = arg;
981 struct drm_i915_gem_object *obj;
982 struct intel_engine_cs *engine;
983 enum intel_engine_id id;
984 struct i915_vma *vma;
989 * If a request takes too long, we would like to give other users
990 * a fair go on the GPU. In particular, users may create batches
991 * that wait upon external input, where that input may even be
992 * supplied by another GPU job. To avoid blocking forever, we
993 * need to preempt the current task and replace it with another
996 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
999 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1001 return PTR_ERR(obj);
1003 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1009 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1010 if (IS_ERR(vaddr)) {
1011 err = PTR_ERR(vaddr);
1015 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1019 err = i915_vma_sync(vma);
1023 for_each_engine(engine, gt, id) {
1024 if (!intel_engine_has_preemption(engine))
1027 memset(vaddr, 0, PAGE_SIZE);
1029 st_engine_heartbeat_disable(engine);
1030 err = slice_semaphore_queue(engine, vma, 5);
1031 st_engine_heartbeat_enable(engine);
1035 if (igt_flush_test(gt->i915)) {
1042 i915_vma_unpin(vma);
1044 i915_gem_object_unpin_map(obj);
1046 i915_gem_object_put(obj);
1050 static struct i915_request *
1051 create_rewinder(struct intel_context *ce,
1052 struct i915_request *wait,
1053 void *slot, int idx)
1056 i915_ggtt_offset(ce->engine->status_page.vma) +
1057 offset_in_page(slot);
1058 struct i915_request *rq;
1062 rq = intel_context_create_request(ce);
1067 err = i915_request_await_dma_fence(rq, &wait->fence);
1072 cs = intel_ring_begin(rq, 14);
1078 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1081 *cs++ = MI_SEMAPHORE_WAIT |
1082 MI_SEMAPHORE_GLOBAL_GTT |
1084 MI_SEMAPHORE_SAD_GTE_SDD;
1089 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1090 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
1091 *cs++ = offset + idx * sizeof(u32);
1094 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1099 intel_ring_advance(rq, cs);
1103 i915_request_get(rq);
1104 i915_request_add(rq);
1106 i915_request_put(rq);
1107 return ERR_PTR(err);
1113 static int live_timeslice_rewind(void *arg)
1115 struct intel_gt *gt = arg;
1116 struct intel_engine_cs *engine;
1117 enum intel_engine_id id;
1120 * The usual presumption on timeslice expiration is that we replace
1121 * the active context with another. However, given a chain of
1122 * dependencies we may end up with replacing the context with itself,
1123 * but only a few of those requests, forcing us to rewind the
1124 * RING_TAIL of the original request.
1126 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1129 for_each_engine(engine, gt, id) {
1130 enum { A1, A2, B1 };
1131 enum { X = 1, Z, Y };
1132 struct i915_request *rq[3] = {};
1133 struct intel_context *ce;
1134 unsigned long timeslice;
1138 if (!intel_engine_has_timeslices(engine))
1142 * A:rq1 -- semaphore wait, timestamp X
1143 * A:rq2 -- write timestamp Y
1145 * B:rq1 [await A:rq1] -- write timestamp Z
1147 * Force timeslice, release semaphore.
1149 * Expect execution/evaluation order XZY
1152 st_engine_heartbeat_disable(engine);
1153 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1155 slot = memset32(engine->status_page.addr + 1000, 0, 4);
1157 ce = intel_context_create(engine);
1163 rq[A1] = create_rewinder(ce, NULL, slot, X);
1164 if (IS_ERR(rq[A1])) {
1165 intel_context_put(ce);
1169 rq[A2] = create_rewinder(ce, NULL, slot, Y);
1170 intel_context_put(ce);
1174 err = wait_for_submit(engine, rq[A2], HZ / 2);
1176 pr_err("%s: failed to submit first context\n",
1181 ce = intel_context_create(engine);
1187 rq[B1] = create_rewinder(ce, rq[A1], slot, Z);
1188 intel_context_put(ce);
1192 err = wait_for_submit(engine, rq[B1], HZ / 2);
1194 pr_err("%s: failed to submit second context\n",
1199 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
1200 ENGINE_TRACE(engine, "forcing tasklet for rewind\n");
1201 while (i915_request_is_active(rq[A2])) { /* semaphore yield! */
1202 /* Wait for the timeslice to kick in */
1203 del_timer(&engine->execlists.timer);
1204 tasklet_hi_schedule(&engine->sched_engine->tasklet);
1205 intel_engine_flush_submission(engine);
1207 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1208 GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1209 GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1210 GEM_BUG_ON(i915_request_is_active(rq[A2]));
1212 /* Release the hounds! */
1214 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1216 for (i = 1; i <= 3; i++) {
1217 unsigned long timeout = jiffies + HZ / 2;
1219 while (!READ_ONCE(slot[i]) &&
1220 time_before(jiffies, timeout))
1223 if (!time_before(jiffies, timeout)) {
1224 pr_err("%s: rq[%d] timed out\n",
1225 engine->name, i - 1);
1230 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1234 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1235 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1243 memset32(&slot[0], -1, 4);
1246 engine->props.timeslice_duration_ms = timeslice;
1247 st_engine_heartbeat_enable(engine);
1248 for (i = 0; i < 3; i++)
1249 i915_request_put(rq[i]);
1250 if (igt_flush_test(gt->i915))
1259 static struct i915_request *nop_request(struct intel_engine_cs *engine)
1261 struct i915_request *rq;
1263 rq = intel_engine_create_kernel_request(engine);
1267 i915_request_get(rq);
1268 i915_request_add(rq);
1273 static long slice_timeout(struct intel_engine_cs *engine)
1277 /* Enough time for a timeslice to kick in, and kick out */
1278 timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));
1280 /* Enough time for the nop request to complete */
1286 static int live_timeslice_queue(void *arg)
1288 struct intel_gt *gt = arg;
1289 struct drm_i915_gem_object *obj;
1290 struct intel_engine_cs *engine;
1291 enum intel_engine_id id;
1292 struct i915_vma *vma;
1297 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1298 * timeslicing between them disabled, we *do* enable timeslicing
1299 * if the queue demands it. (Normally, we do not submit if
1300 * ELSP[1] is already occupied, so must rely on timeslicing to
1301 * eject ELSP[0] in favour of the queue.)
1303 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1306 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1308 return PTR_ERR(obj);
1310 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1316 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1317 if (IS_ERR(vaddr)) {
1318 err = PTR_ERR(vaddr);
1322 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1326 err = i915_vma_sync(vma);
1330 for_each_engine(engine, gt, id) {
1331 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
1332 struct i915_request *rq, *nop;
1334 if (!intel_engine_has_preemption(engine))
1337 st_engine_heartbeat_disable(engine);
1338 memset(vaddr, 0, PAGE_SIZE);
1340 /* ELSP[0]: semaphore wait */
1341 rq = semaphore_queue(engine, vma, 0);
1346 engine->sched_engine->schedule(rq, &attr);
1347 err = wait_for_submit(engine, rq, HZ / 2);
1349 pr_err("%s: Timed out trying to submit semaphores\n",
1354 /* ELSP[1]: nop request */
1355 nop = nop_request(engine);
1360 err = wait_for_submit(engine, nop, HZ / 2);
1361 i915_request_put(nop);
1363 pr_err("%s: Timed out trying to submit nop\n",
1368 GEM_BUG_ON(i915_request_completed(rq));
1369 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1371 /* Queue: semaphore signal, matching priority as semaphore */
1372 err = release_queue(engine, vma, 1, effective_prio(rq));
1376 /* Wait until we ack the release_queue and start timeslicing */
1379 intel_engine_flush_submission(engine);
1380 } while (READ_ONCE(engine->execlists.pending[0]));
1382 /* Timeslice every jiffy, so within 2 we should signal */
1383 if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {
1384 struct drm_printer p =
1385 drm_info_printer(gt->i915->drm.dev);
1387 pr_err("%s: Failed to timeslice into queue\n",
1389 intel_engine_dump(engine, &p,
1390 "%s\n", engine->name);
1392 memset(vaddr, 0xff, PAGE_SIZE);
1396 i915_request_put(rq);
1398 st_engine_heartbeat_enable(engine);
1404 i915_vma_unpin(vma);
1406 i915_gem_object_unpin_map(obj);
1408 i915_gem_object_put(obj);
1412 static int live_timeslice_nopreempt(void *arg)
1414 struct intel_gt *gt = arg;
1415 struct intel_engine_cs *engine;
1416 enum intel_engine_id id;
1417 struct igt_spinner spin;
1421 * We should not timeslice into a request that is marked with
1422 * I915_REQUEST_NOPREEMPT.
1424 if (!CONFIG_DRM_I915_TIMESLICE_DURATION)
1427 if (igt_spinner_init(&spin, gt))
1430 for_each_engine(engine, gt, id) {
1431 struct intel_context *ce;
1432 struct i915_request *rq;
1433 unsigned long timeslice;
1435 if (!intel_engine_has_preemption(engine))
1438 ce = intel_context_create(engine);
1444 st_engine_heartbeat_disable(engine);
1445 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1447 /* Create an unpreemptible spinner */
1449 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
1450 intel_context_put(ce);
1456 i915_request_get(rq);
1457 i915_request_add(rq);
1459 if (!igt_wait_for_spinner(&spin, rq)) {
1460 i915_request_put(rq);
1465 set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
1466 i915_request_put(rq);
1468 /* Followed by a maximum priority barrier (heartbeat) */
1470 ce = intel_context_create(engine);
1476 rq = intel_context_create_request(ce);
1477 intel_context_put(ce);
1483 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1484 i915_request_get(rq);
1485 i915_request_add(rq);
1488 * Wait until the barrier is in ELSP, and we know timeslicing
1489 * will have been activated.
1491 if (wait_for_submit(engine, rq, HZ / 2)) {
1492 i915_request_put(rq);
1498 * Since the ELSP[0] request is unpreemptible, it should not
1499 * allow the maximum priority barrier through. Wait long
1500 * enough to see if it is timesliced in by mistake.
1502 if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {
1503 pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
1507 i915_request_put(rq);
1510 igt_spinner_end(&spin);
1512 xchg(&engine->props.timeslice_duration_ms, timeslice);
1513 st_engine_heartbeat_enable(engine);
1517 if (igt_flush_test(gt->i915)) {
1523 igt_spinner_fini(&spin);
1527 static int live_busywait_preempt(void *arg)
1529 struct intel_gt *gt = arg;
1530 struct i915_gem_context *ctx_hi, *ctx_lo;
1531 struct intel_engine_cs *engine;
1532 struct drm_i915_gem_object *obj;
1533 struct i915_vma *vma;
1534 enum intel_engine_id id;
1539 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1540 * preempt the busywaits used to synchronise between rings.
1543 ctx_hi = kernel_context(gt->i915, NULL);
1546 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1548 ctx_lo = kernel_context(gt->i915, NULL);
1551 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1553 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1559 map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1565 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1571 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1575 err = i915_vma_sync(vma);
1579 for_each_engine(engine, gt, id) {
1580 struct i915_request *lo, *hi;
1581 struct igt_live_test t;
1584 if (!intel_engine_has_preemption(engine))
1587 if (!intel_engine_can_store_dword(engine))
1590 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1596 * We create two requests. The low priority request
1597 * busywaits on a semaphore (inside the ringbuffer where
1598 * is should be preemptible) and the high priority requests
1599 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1600 * allowing the first request to complete. If preemption
1601 * fails, we hang instead.
1604 lo = igt_request_alloc(ctx_lo, engine);
1610 cs = intel_ring_begin(lo, 8);
1613 i915_request_add(lo);
1617 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1618 *cs++ = i915_ggtt_offset(vma);
1622 /* XXX Do we need a flush + invalidate here? */
1624 *cs++ = MI_SEMAPHORE_WAIT |
1625 MI_SEMAPHORE_GLOBAL_GTT |
1627 MI_SEMAPHORE_SAD_EQ_SDD;
1629 *cs++ = i915_ggtt_offset(vma);
1632 intel_ring_advance(lo, cs);
1634 i915_request_get(lo);
1635 i915_request_add(lo);
1637 if (wait_for(READ_ONCE(*map), 10)) {
1638 i915_request_put(lo);
1643 /* Low priority request should be busywaiting now */
1644 if (i915_request_wait(lo, 0, 1) != -ETIME) {
1645 i915_request_put(lo);
1646 pr_err("%s: Busywaiting request did not!\n",
1652 hi = igt_request_alloc(ctx_hi, engine);
1655 i915_request_put(lo);
1659 cs = intel_ring_begin(hi, 4);
1662 i915_request_add(hi);
1663 i915_request_put(lo);
1667 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1668 *cs++ = i915_ggtt_offset(vma);
1672 intel_ring_advance(hi, cs);
1673 i915_request_add(hi);
1675 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1676 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1678 pr_err("%s: Failed to preempt semaphore busywait!\n",
1681 intel_engine_dump(engine, &p, "%s\n", engine->name);
1684 i915_request_put(lo);
1685 intel_gt_set_wedged(gt);
1689 GEM_BUG_ON(READ_ONCE(*map));
1690 i915_request_put(lo);
1692 if (igt_live_test_end(&t)) {
1700 i915_vma_unpin(vma);
1702 i915_gem_object_unpin_map(obj);
1704 i915_gem_object_put(obj);
1706 kernel_context_close(ctx_lo);
1708 kernel_context_close(ctx_hi);
1712 static struct i915_request *
1713 spinner_create_request(struct igt_spinner *spin,
1714 struct i915_gem_context *ctx,
1715 struct intel_engine_cs *engine,
1718 struct intel_context *ce;
1719 struct i915_request *rq;
1721 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1723 return ERR_CAST(ce);
1725 rq = igt_spinner_create_request(spin, ce, arb);
1726 intel_context_put(ce);
1730 static int live_preempt(void *arg)
1732 struct intel_gt *gt = arg;
1733 struct i915_gem_context *ctx_hi, *ctx_lo;
1734 struct igt_spinner spin_hi, spin_lo;
1735 struct intel_engine_cs *engine;
1736 enum intel_engine_id id;
1739 if (igt_spinner_init(&spin_hi, gt))
1742 if (igt_spinner_init(&spin_lo, gt))
1745 ctx_hi = kernel_context(gt->i915, NULL);
1748 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1750 ctx_lo = kernel_context(gt->i915, NULL);
1753 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1755 for_each_engine(engine, gt, id) {
1756 struct igt_live_test t;
1757 struct i915_request *rq;
1759 if (!intel_engine_has_preemption(engine))
1762 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1767 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1774 i915_request_add(rq);
1775 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1776 GEM_TRACE("lo spinner failed to start\n");
1778 intel_gt_set_wedged(gt);
1783 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1786 igt_spinner_end(&spin_lo);
1791 i915_request_add(rq);
1792 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1793 GEM_TRACE("hi spinner failed to start\n");
1795 intel_gt_set_wedged(gt);
1800 igt_spinner_end(&spin_hi);
1801 igt_spinner_end(&spin_lo);
1803 if (igt_live_test_end(&t)) {
1811 kernel_context_close(ctx_lo);
1813 kernel_context_close(ctx_hi);
1815 igt_spinner_fini(&spin_lo);
1817 igt_spinner_fini(&spin_hi);
1821 static int live_late_preempt(void *arg)
1823 struct intel_gt *gt = arg;
1824 struct i915_gem_context *ctx_hi, *ctx_lo;
1825 struct igt_spinner spin_hi, spin_lo;
1826 struct intel_engine_cs *engine;
1827 struct i915_sched_attr attr = {};
1828 enum intel_engine_id id;
1831 if (igt_spinner_init(&spin_hi, gt))
1834 if (igt_spinner_init(&spin_lo, gt))
1837 ctx_hi = kernel_context(gt->i915, NULL);
1841 ctx_lo = kernel_context(gt->i915, NULL);
1845 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1846 ctx_lo->sched.priority = 1;
1848 for_each_engine(engine, gt, id) {
1849 struct igt_live_test t;
1850 struct i915_request *rq;
1852 if (!intel_engine_has_preemption(engine))
1855 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1860 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1867 i915_request_add(rq);
1868 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1869 pr_err("First context failed to start\n");
1873 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1876 igt_spinner_end(&spin_lo);
1881 i915_request_add(rq);
1882 if (igt_wait_for_spinner(&spin_hi, rq)) {
1883 pr_err("Second context overtook first?\n");
1887 attr.priority = I915_PRIORITY_MAX;
1888 engine->sched_engine->schedule(rq, &attr);
1890 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1891 pr_err("High priority context failed to preempt the low priority context\n");
1896 igt_spinner_end(&spin_hi);
1897 igt_spinner_end(&spin_lo);
1899 if (igt_live_test_end(&t)) {
1907 kernel_context_close(ctx_lo);
1909 kernel_context_close(ctx_hi);
1911 igt_spinner_fini(&spin_lo);
1913 igt_spinner_fini(&spin_hi);
1917 igt_spinner_end(&spin_hi);
1918 igt_spinner_end(&spin_lo);
1919 intel_gt_set_wedged(gt);
1924 struct preempt_client {
1925 struct igt_spinner spin;
1926 struct i915_gem_context *ctx;
1929 static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1931 c->ctx = kernel_context(gt->i915, NULL);
1935 if (igt_spinner_init(&c->spin, gt))
1941 kernel_context_close(c->ctx);
1945 static void preempt_client_fini(struct preempt_client *c)
1947 igt_spinner_fini(&c->spin);
1948 kernel_context_close(c->ctx);
1951 static int live_nopreempt(void *arg)
1953 struct intel_gt *gt = arg;
1954 struct intel_engine_cs *engine;
1955 struct preempt_client a, b;
1956 enum intel_engine_id id;
1960 * Verify that we can disable preemption for an individual request
1961 * that may be being observed and not want to be interrupted.
1964 if (preempt_client_init(gt, &a))
1966 if (preempt_client_init(gt, &b))
1968 b.ctx->sched.priority = I915_PRIORITY_MAX;
1970 for_each_engine(engine, gt, id) {
1971 struct i915_request *rq_a, *rq_b;
1973 if (!intel_engine_has_preemption(engine))
1976 engine->execlists.preempt_hang.count = 0;
1978 rq_a = spinner_create_request(&a.spin,
1982 err = PTR_ERR(rq_a);
1986 /* Low priority client, but unpreemptable! */
1987 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1989 i915_request_add(rq_a);
1990 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1991 pr_err("First client failed to start\n");
1995 rq_b = spinner_create_request(&b.spin,
1999 err = PTR_ERR(rq_b);
2003 i915_request_add(rq_b);
2005 /* B is much more important than A! (But A is unpreemptable.) */
2006 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
2008 /* Wait long enough for preemption and timeslicing */
2009 if (igt_wait_for_spinner(&b.spin, rq_b)) {
2010 pr_err("Second client started too early!\n");
2014 igt_spinner_end(&a.spin);
2016 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2017 pr_err("Second client failed to start\n");
2021 igt_spinner_end(&b.spin);
2023 if (engine->execlists.preempt_hang.count) {
2024 pr_err("Preemption recorded x%d; should have been suppressed!\n",
2025 engine->execlists.preempt_hang.count);
2030 if (igt_flush_test(gt->i915))
2036 preempt_client_fini(&b);
2038 preempt_client_fini(&a);
2042 igt_spinner_end(&b.spin);
2043 igt_spinner_end(&a.spin);
2044 intel_gt_set_wedged(gt);
2049 struct live_preempt_cancel {
2050 struct intel_engine_cs *engine;
2051 struct preempt_client a, b;
2054 static int __cancel_active0(struct live_preempt_cancel *arg)
2056 struct i915_request *rq;
2057 struct igt_live_test t;
2060 /* Preempt cancel of ELSP0 */
2061 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2062 if (igt_live_test_begin(&t, arg->engine->i915,
2063 __func__, arg->engine->name))
2066 rq = spinner_create_request(&arg->a.spin,
2067 arg->a.ctx, arg->engine,
2072 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2073 i915_request_get(rq);
2074 i915_request_add(rq);
2075 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2080 intel_context_set_banned(rq->context);
2081 err = intel_engine_pulse(arg->engine);
2085 err = wait_for_reset(arg->engine, rq, HZ / 2);
2087 pr_err("Cancelled inflight0 request did not reset\n");
2092 i915_request_put(rq);
2093 if (igt_live_test_end(&t))
2098 static int __cancel_active1(struct live_preempt_cancel *arg)
2100 struct i915_request *rq[2] = {};
2101 struct igt_live_test t;
2104 /* Preempt cancel of ELSP1 */
2105 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2106 if (igt_live_test_begin(&t, arg->engine->i915,
2107 __func__, arg->engine->name))
2110 rq[0] = spinner_create_request(&arg->a.spin,
2111 arg->a.ctx, arg->engine,
2112 MI_NOOP); /* no preemption */
2114 return PTR_ERR(rq[0]);
2116 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2117 i915_request_get(rq[0]);
2118 i915_request_add(rq[0]);
2119 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2124 rq[1] = spinner_create_request(&arg->b.spin,
2125 arg->b.ctx, arg->engine,
2127 if (IS_ERR(rq[1])) {
2128 err = PTR_ERR(rq[1]);
2132 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2133 i915_request_get(rq[1]);
2134 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2135 i915_request_add(rq[1]);
2139 intel_context_set_banned(rq[1]->context);
2140 err = intel_engine_pulse(arg->engine);
2144 igt_spinner_end(&arg->a.spin);
2145 err = wait_for_reset(arg->engine, rq[1], HZ / 2);
2149 if (rq[0]->fence.error != 0) {
2150 pr_err("Normal inflight0 request did not complete\n");
2155 if (rq[1]->fence.error != -EIO) {
2156 pr_err("Cancelled inflight1 request did not report -EIO\n");
2162 i915_request_put(rq[1]);
2163 i915_request_put(rq[0]);
2164 if (igt_live_test_end(&t))
2169 static int __cancel_queued(struct live_preempt_cancel *arg)
2171 struct i915_request *rq[3] = {};
2172 struct igt_live_test t;
2175 /* Full ELSP and one in the wings */
2176 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2177 if (igt_live_test_begin(&t, arg->engine->i915,
2178 __func__, arg->engine->name))
2181 rq[0] = spinner_create_request(&arg->a.spin,
2182 arg->a.ctx, arg->engine,
2185 return PTR_ERR(rq[0]);
2187 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2188 i915_request_get(rq[0]);
2189 i915_request_add(rq[0]);
2190 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2195 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
2196 if (IS_ERR(rq[1])) {
2197 err = PTR_ERR(rq[1]);
2201 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2202 i915_request_get(rq[1]);
2203 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2204 i915_request_add(rq[1]);
2208 rq[2] = spinner_create_request(&arg->b.spin,
2209 arg->a.ctx, arg->engine,
2211 if (IS_ERR(rq[2])) {
2212 err = PTR_ERR(rq[2]);
2216 i915_request_get(rq[2]);
2217 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
2218 i915_request_add(rq[2]);
2222 intel_context_set_banned(rq[2]->context);
2223 err = intel_engine_pulse(arg->engine);
2227 err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2231 if (rq[0]->fence.error != -EIO) {
2232 pr_err("Cancelled inflight0 request did not report -EIO\n");
2237 if (rq[1]->fence.error != 0) {
2238 pr_err("Normal inflight1 request did not complete\n");
2243 if (rq[2]->fence.error != -EIO) {
2244 pr_err("Cancelled queued request did not report -EIO\n");
2250 i915_request_put(rq[2]);
2251 i915_request_put(rq[1]);
2252 i915_request_put(rq[0]);
2253 if (igt_live_test_end(&t))
2258 static int __cancel_hostile(struct live_preempt_cancel *arg)
2260 struct i915_request *rq;
2263 /* Preempt cancel non-preemptible spinner in ELSP0 */
2264 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2267 if (!intel_has_reset_engine(arg->engine->gt))
2270 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2271 rq = spinner_create_request(&arg->a.spin,
2272 arg->a.ctx, arg->engine,
2273 MI_NOOP); /* preemption disabled */
2277 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2278 i915_request_get(rq);
2279 i915_request_add(rq);
2280 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2285 intel_context_set_banned(rq->context);
2286 err = intel_engine_pulse(arg->engine); /* force reset */
2290 err = wait_for_reset(arg->engine, rq, HZ / 2);
2292 pr_err("Cancelled inflight0 request did not reset\n");
2297 i915_request_put(rq);
2298 if (igt_flush_test(arg->engine->i915))
2303 static void force_reset_timeout(struct intel_engine_cs *engine)
2305 engine->reset_timeout.probability = 999;
2306 atomic_set(&engine->reset_timeout.times, -1);
2309 static void cancel_reset_timeout(struct intel_engine_cs *engine)
2311 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
2314 static int __cancel_fail(struct live_preempt_cancel *arg)
2316 struct intel_engine_cs *engine = arg->engine;
2317 struct i915_request *rq;
2320 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2323 if (!intel_has_reset_engine(engine->gt))
2326 GEM_TRACE("%s(%s)\n", __func__, engine->name);
2327 rq = spinner_create_request(&arg->a.spin,
2329 MI_NOOP); /* preemption disabled */
2333 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2334 i915_request_get(rq);
2335 i915_request_add(rq);
2336 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2341 intel_context_set_banned(rq->context);
2343 err = intel_engine_pulse(engine);
2347 force_reset_timeout(engine);
2349 /* force preempt reset [failure] */
2350 while (!engine->execlists.pending[0])
2351 intel_engine_flush_submission(engine);
2352 del_timer_sync(&engine->execlists.preempt);
2353 intel_engine_flush_submission(engine);
2355 cancel_reset_timeout(engine);
2357 /* after failure, require heartbeats to reset device */
2358 intel_engine_set_heartbeat(engine, 1);
2359 err = wait_for_reset(engine, rq, HZ / 2);
2360 intel_engine_set_heartbeat(engine,
2361 engine->defaults.heartbeat_interval_ms);
2363 pr_err("Cancelled inflight0 request did not reset\n");
2368 i915_request_put(rq);
2369 if (igt_flush_test(engine->i915))
2374 static int live_preempt_cancel(void *arg)
2376 struct intel_gt *gt = arg;
2377 struct live_preempt_cancel data;
2378 enum intel_engine_id id;
2382 * To cancel an inflight context, we need to first remove it from the
2383 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2386 if (preempt_client_init(gt, &data.a))
2388 if (preempt_client_init(gt, &data.b))
2391 for_each_engine(data.engine, gt, id) {
2392 if (!intel_engine_has_preemption(data.engine))
2395 err = __cancel_active0(&data);
2399 err = __cancel_active1(&data);
2403 err = __cancel_queued(&data);
2407 err = __cancel_hostile(&data);
2411 err = __cancel_fail(&data);
2418 preempt_client_fini(&data.b);
2420 preempt_client_fini(&data.a);
2425 igt_spinner_end(&data.b.spin);
2426 igt_spinner_end(&data.a.spin);
2427 intel_gt_set_wedged(gt);
2431 static int live_suppress_self_preempt(void *arg)
2433 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2434 struct intel_gt *gt = arg;
2435 struct intel_engine_cs *engine;
2436 struct preempt_client a, b;
2437 enum intel_engine_id id;
2441 * Verify that if a preemption request does not cause a change in
2442 * the current execution order, the preempt-to-idle injection is
2443 * skipped and that we do not accidentally apply it after the CS
2447 if (intel_uc_uses_guc_submission(>->uc))
2448 return 0; /* presume black blox */
2450 if (intel_vgpu_active(gt->i915))
2451 return 0; /* GVT forces single port & request submission */
2453 if (preempt_client_init(gt, &a))
2455 if (preempt_client_init(gt, &b))
2458 for_each_engine(engine, gt, id) {
2459 struct i915_request *rq_a, *rq_b;
2462 if (!intel_engine_has_preemption(engine))
2465 if (igt_flush_test(gt->i915))
2468 st_engine_heartbeat_disable(engine);
2469 engine->execlists.preempt_hang.count = 0;
2471 rq_a = spinner_create_request(&a.spin,
2475 err = PTR_ERR(rq_a);
2476 st_engine_heartbeat_enable(engine);
2480 i915_request_add(rq_a);
2481 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2482 pr_err("First client failed to start\n");
2483 st_engine_heartbeat_enable(engine);
2487 /* Keep postponing the timer to avoid premature slicing */
2488 mod_timer(&engine->execlists.timer, jiffies + HZ);
2489 for (depth = 0; depth < 8; depth++) {
2490 rq_b = spinner_create_request(&b.spin,
2494 err = PTR_ERR(rq_b);
2495 st_engine_heartbeat_enable(engine);
2498 i915_request_add(rq_b);
2500 GEM_BUG_ON(i915_request_completed(rq_a));
2501 engine->sched_engine->schedule(rq_a, &attr);
2502 igt_spinner_end(&a.spin);
2504 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2505 pr_err("Second client failed to start\n");
2506 st_engine_heartbeat_enable(engine);
2513 igt_spinner_end(&a.spin);
2515 if (engine->execlists.preempt_hang.count) {
2516 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2518 engine->execlists.preempt_hang.count,
2520 st_engine_heartbeat_enable(engine);
2525 st_engine_heartbeat_enable(engine);
2526 if (igt_flush_test(gt->i915))
2532 preempt_client_fini(&b);
2534 preempt_client_fini(&a);
2538 igt_spinner_end(&b.spin);
2539 igt_spinner_end(&a.spin);
2540 intel_gt_set_wedged(gt);
2545 static int live_chain_preempt(void *arg)
2547 struct intel_gt *gt = arg;
2548 struct intel_engine_cs *engine;
2549 struct preempt_client hi, lo;
2550 enum intel_engine_id id;
2554 * Build a chain AB...BA between two contexts (A, B) and request
2555 * preemption of the last request. It should then complete before
2556 * the previously submitted spinner in B.
2559 if (preempt_client_init(gt, &hi))
2562 if (preempt_client_init(gt, &lo))
2565 for_each_engine(engine, gt, id) {
2566 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2567 struct igt_live_test t;
2568 struct i915_request *rq;
2569 int ring_size, count, i;
2571 if (!intel_engine_has_preemption(engine))
2574 rq = spinner_create_request(&lo.spin,
2580 i915_request_get(rq);
2581 i915_request_add(rq);
2583 ring_size = rq->wa_tail - rq->head;
2585 ring_size += rq->ring->size;
2586 ring_size = rq->ring->size / ring_size;
2587 pr_debug("%s(%s): Using maximum of %d requests\n",
2588 __func__, engine->name, ring_size);
2590 igt_spinner_end(&lo.spin);
2591 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2592 pr_err("Timed out waiting to flush %s\n", engine->name);
2593 i915_request_put(rq);
2596 i915_request_put(rq);
2598 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2603 for_each_prime_number_from(count, 1, ring_size) {
2604 rq = spinner_create_request(&hi.spin,
2609 i915_request_add(rq);
2610 if (!igt_wait_for_spinner(&hi.spin, rq))
2613 rq = spinner_create_request(&lo.spin,
2618 i915_request_add(rq);
2620 for (i = 0; i < count; i++) {
2621 rq = igt_request_alloc(lo.ctx, engine);
2624 i915_request_add(rq);
2627 rq = igt_request_alloc(hi.ctx, engine);
2631 i915_request_get(rq);
2632 i915_request_add(rq);
2633 engine->sched_engine->schedule(rq, &attr);
2635 igt_spinner_end(&hi.spin);
2636 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2637 struct drm_printer p =
2638 drm_info_printer(gt->i915->drm.dev);
2640 pr_err("Failed to preempt over chain of %d\n",
2642 intel_engine_dump(engine, &p,
2643 "%s\n", engine->name);
2644 i915_request_put(rq);
2647 igt_spinner_end(&lo.spin);
2648 i915_request_put(rq);
2650 rq = igt_request_alloc(lo.ctx, engine);
2654 i915_request_get(rq);
2655 i915_request_add(rq);
2657 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2658 struct drm_printer p =
2659 drm_info_printer(gt->i915->drm.dev);
2661 pr_err("Failed to flush low priority chain of %d requests\n",
2663 intel_engine_dump(engine, &p,
2664 "%s\n", engine->name);
2666 i915_request_put(rq);
2669 i915_request_put(rq);
2672 if (igt_live_test_end(&t)) {
2680 preempt_client_fini(&lo);
2682 preempt_client_fini(&hi);
2686 igt_spinner_end(&hi.spin);
2687 igt_spinner_end(&lo.spin);
2688 intel_gt_set_wedged(gt);
2693 static int create_gang(struct intel_engine_cs *engine,
2694 struct i915_request **prev)
2696 struct drm_i915_gem_object *obj;
2697 struct intel_context *ce;
2698 struct i915_request *rq;
2699 struct i915_vma *vma;
2703 ce = intel_context_create(engine);
2707 obj = i915_gem_object_create_internal(engine->i915, 4096);
2713 vma = i915_vma_instance(obj, ce->vm, NULL);
2719 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2723 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
2729 /* Semaphore target: spin until zero */
2730 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2732 *cs++ = MI_SEMAPHORE_WAIT |
2734 MI_SEMAPHORE_SAD_EQ_SDD;
2736 *cs++ = lower_32_bits(vma->node.start);
2737 *cs++ = upper_32_bits(vma->node.start);
2740 u64 offset = (*prev)->batch->node.start;
2742 /* Terminate the spinner in the next lower priority batch. */
2743 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2744 *cs++ = lower_32_bits(offset);
2745 *cs++ = upper_32_bits(offset);
2749 *cs++ = MI_BATCH_BUFFER_END;
2750 i915_gem_object_flush_map(obj);
2751 i915_gem_object_unpin_map(obj);
2753 rq = intel_context_create_request(ce);
2759 rq->batch = i915_vma_get(vma);
2760 i915_request_get(rq);
2763 err = i915_request_await_object(rq, vma->obj, false);
2765 err = i915_vma_move_to_active(vma, rq, 0);
2767 err = rq->engine->emit_bb_start(rq,
2770 i915_vma_unlock(vma);
2771 i915_request_add(rq);
2775 i915_gem_object_put(obj);
2776 intel_context_put(ce);
2778 rq->mock.link.next = &(*prev)->mock.link;
2783 i915_vma_put(rq->batch);
2784 i915_request_put(rq);
2786 i915_gem_object_put(obj);
2788 intel_context_put(ce);
2792 static int __live_preempt_ring(struct intel_engine_cs *engine,
2793 struct igt_spinner *spin,
2794 int queue_sz, int ring_sz)
2796 struct intel_context *ce[2] = {};
2797 struct i915_request *rq;
2798 struct igt_live_test t;
2802 if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))
2805 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2806 struct intel_context *tmp;
2808 tmp = intel_context_create(engine);
2814 tmp->ring_size = ring_sz;
2816 err = intel_context_pin(tmp);
2818 intel_context_put(tmp);
2822 memset32(tmp->ring->vaddr,
2823 0xdeadbeef, /* trigger a hang if executed */
2824 tmp->ring->vma->size / sizeof(u32));
2829 rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);
2835 i915_request_get(rq);
2836 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2837 i915_request_add(rq);
2839 if (!igt_wait_for_spinner(spin, rq)) {
2840 intel_gt_set_wedged(engine->gt);
2841 i915_request_put(rq);
2846 /* Fill the ring, until we will cause a wrap */
2848 while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {
2849 struct i915_request *tmp;
2851 tmp = intel_context_create_request(ce[0]);
2854 i915_request_put(rq);
2858 i915_request_add(tmp);
2859 intel_engine_flush_submission(engine);
2862 intel_engine_flush_submission(engine);
2863 pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
2864 engine->name, queue_sz, n,
2869 i915_request_put(rq);
2871 /* Create a second request to preempt the first ring */
2872 rq = intel_context_create_request(ce[1]);
2878 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2879 i915_request_get(rq);
2880 i915_request_add(rq);
2882 err = wait_for_submit(engine, rq, HZ / 2);
2883 i915_request_put(rq);
2885 pr_err("%s: preemption request was not submitted\n",
2890 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
2892 ce[0]->ring->tail, ce[0]->ring->emit,
2893 ce[1]->ring->tail, ce[1]->ring->emit);
2896 intel_engine_flush_submission(engine);
2897 igt_spinner_end(spin);
2898 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2899 if (IS_ERR_OR_NULL(ce[n]))
2902 intel_context_unpin(ce[n]);
2903 intel_context_put(ce[n]);
2905 if (igt_live_test_end(&t))
2910 static int live_preempt_ring(void *arg)
2912 struct intel_gt *gt = arg;
2913 struct intel_engine_cs *engine;
2914 struct igt_spinner spin;
2915 enum intel_engine_id id;
2919 * Check that we rollback large chunks of a ring in order to do a
2920 * preemption event. Similar to live_unlite_ring, but looking at
2921 * ring size rather than the impact of intel_ring_direction().
2924 if (igt_spinner_init(&spin, gt))
2927 for_each_engine(engine, gt, id) {
2930 if (!intel_engine_has_preemption(engine))
2933 if (!intel_engine_can_store_dword(engine))
2936 st_engine_heartbeat_disable(engine);
2938 for (n = 0; n <= 3; n++) {
2939 err = __live_preempt_ring(engine, &spin,
2940 n * SZ_4K / 4, SZ_4K);
2945 st_engine_heartbeat_enable(engine);
2950 igt_spinner_fini(&spin);
2954 static int live_preempt_gang(void *arg)
2956 struct intel_gt *gt = arg;
2957 struct intel_engine_cs *engine;
2958 enum intel_engine_id id;
2961 * Build as long a chain of preempters as we can, with each
2962 * request higher priority than the last. Once we are ready, we release
2963 * the last batch which then precolates down the chain, each releasing
2964 * the next oldest in turn. The intent is to simply push as hard as we
2965 * can with the number of preemptions, trying to exceed narrow HW
2966 * limits. At a minimum, we insist that we can sort all the user
2967 * high priority levels into execution order.
2970 for_each_engine(engine, gt, id) {
2971 struct i915_request *rq = NULL;
2972 struct igt_live_test t;
2973 IGT_TIMEOUT(end_time);
2978 if (!intel_engine_has_preemption(engine))
2981 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2985 struct i915_sched_attr attr = { .priority = prio++ };
2987 err = create_gang(engine, &rq);
2991 /* Submit each spinner at increasing priority */
2992 engine->sched_engine->schedule(rq, &attr);
2993 } while (prio <= I915_PRIORITY_MAX &&
2994 !__igt_timeout(end_time, NULL));
2995 pr_debug("%s: Preempt chain of %d requests\n",
2996 engine->name, prio);
2999 * Such that the last spinner is the highest priority and
3000 * should execute first. When that spinner completes,
3001 * it will terminate the next lowest spinner until there
3002 * are no more spinners and the gang is complete.
3004 cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
3007 i915_gem_object_unpin_map(rq->batch->obj);
3010 intel_gt_set_wedged(gt);
3013 while (rq) { /* wait for each rq from highest to lowest prio */
3014 struct i915_request *n = list_next_entry(rq, mock.link);
3016 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
3017 struct drm_printer p =
3018 drm_info_printer(engine->i915->drm.dev);
3020 pr_err("Failed to flush chain of %d requests, at %d\n",
3022 intel_engine_dump(engine, &p,
3023 "%s\n", engine->name);
3028 i915_vma_put(rq->batch);
3029 i915_request_put(rq);
3033 if (igt_live_test_end(&t))
3042 static struct i915_vma *
3043 create_gpr_user(struct intel_engine_cs *engine,
3044 struct i915_vma *result,
3045 unsigned int offset)
3047 struct drm_i915_gem_object *obj;
3048 struct i915_vma *vma;
3053 obj = i915_gem_object_create_internal(engine->i915, 4096);
3055 return ERR_CAST(obj);
3057 vma = i915_vma_instance(obj, result->vm, NULL);
3059 i915_gem_object_put(obj);
3063 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3066 return ERR_PTR(err);
3069 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
3072 return ERR_CAST(cs);
3075 /* All GPR are clear for new contexts. We use GPR(0) as a constant */
3076 *cs++ = MI_LOAD_REGISTER_IMM(1);
3077 *cs++ = CS_GPR(engine, 0);
3080 for (i = 1; i < NUM_GPR; i++) {
3086 * As we read and write into the context saved GPR[i], if
3087 * we restart this batch buffer from an earlier point, we
3088 * will repeat the increment and store a value > 1.
3091 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
3092 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
3093 *cs++ = MI_MATH_ADD;
3094 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
3096 addr = result->node.start + offset + i * sizeof(*cs);
3097 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
3098 *cs++ = CS_GPR(engine, 2 * i);
3099 *cs++ = lower_32_bits(addr);
3100 *cs++ = upper_32_bits(addr);
3102 *cs++ = MI_SEMAPHORE_WAIT |
3104 MI_SEMAPHORE_SAD_GTE_SDD;
3106 *cs++ = lower_32_bits(result->node.start);
3107 *cs++ = upper_32_bits(result->node.start);
3110 *cs++ = MI_BATCH_BUFFER_END;
3111 i915_gem_object_flush_map(obj);
3112 i915_gem_object_unpin_map(obj);
3117 static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
3119 struct drm_i915_gem_object *obj;
3120 struct i915_vma *vma;
3123 obj = i915_gem_object_create_internal(gt->i915, sz);
3125 return ERR_CAST(obj);
3127 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
3129 i915_gem_object_put(obj);
3133 err = i915_ggtt_pin(vma, NULL, 0, 0);
3136 return ERR_PTR(err);
3142 static struct i915_request *
3143 create_gpr_client(struct intel_engine_cs *engine,
3144 struct i915_vma *global,
3145 unsigned int offset)
3147 struct i915_vma *batch, *vma;
3148 struct intel_context *ce;
3149 struct i915_request *rq;
3152 ce = intel_context_create(engine);
3154 return ERR_CAST(ce);
3156 vma = i915_vma_instance(global->obj, ce->vm, NULL);
3162 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3166 batch = create_gpr_user(engine, vma, offset);
3167 if (IS_ERR(batch)) {
3168 err = PTR_ERR(batch);
3172 rq = intel_context_create_request(ce);
3179 err = i915_request_await_object(rq, vma->obj, false);
3181 err = i915_vma_move_to_active(vma, rq, 0);
3182 i915_vma_unlock(vma);
3184 i915_vma_lock(batch);
3186 err = i915_request_await_object(rq, batch->obj, false);
3188 err = i915_vma_move_to_active(batch, rq, 0);
3190 err = rq->engine->emit_bb_start(rq,
3193 i915_vma_unlock(batch);
3194 i915_vma_unpin(batch);
3197 i915_request_get(rq);
3198 i915_request_add(rq);
3201 i915_vma_put(batch);
3203 i915_vma_unpin(vma);
3205 intel_context_put(ce);
3206 return err ? ERR_PTR(err) : rq;
3209 static int preempt_user(struct intel_engine_cs *engine,
3210 struct i915_vma *global,
3213 struct i915_sched_attr attr = {
3214 .priority = I915_PRIORITY_MAX
3216 struct i915_request *rq;
3220 rq = intel_engine_create_kernel_request(engine);
3224 cs = intel_ring_begin(rq, 4);
3226 i915_request_add(rq);
3230 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
3231 *cs++ = i915_ggtt_offset(global);
3235 intel_ring_advance(rq, cs);
3237 i915_request_get(rq);
3238 i915_request_add(rq);
3240 engine->sched_engine->schedule(rq, &attr);
3242 if (i915_request_wait(rq, 0, HZ / 2) < 0)
3244 i915_request_put(rq);
3249 static int live_preempt_user(void *arg)
3251 struct intel_gt *gt = arg;
3252 struct intel_engine_cs *engine;
3253 struct i915_vma *global;
3254 enum intel_engine_id id;
3259 * In our other tests, we look at preemption in carefully
3260 * controlled conditions in the ringbuffer. Since most of the
3261 * time is spent in user batches, most of our preemptions naturally
3262 * occur there. We want to verify that when we preempt inside a batch
3263 * we continue on from the current instruction and do not roll back
3264 * to the start, or another earlier arbitration point.
3266 * To verify this, we create a batch which is a mixture of
3267 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with
3268 * a few preempting contexts thrown into the mix, we look for any
3269 * repeated instructions (which show up as incorrect values).
3272 global = create_global(gt, 4096);
3274 return PTR_ERR(global);
3276 result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
3277 if (IS_ERR(result)) {
3278 i915_vma_unpin_and_release(&global, 0);
3279 return PTR_ERR(result);
3282 for_each_engine(engine, gt, id) {
3283 struct i915_request *client[3] = {};
3284 struct igt_live_test t;
3287 if (!intel_engine_has_preemption(engine))
3290 if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS)
3291 continue; /* we need per-context GPR */
3293 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3298 memset(result, 0, 4096);
3300 for (i = 0; i < ARRAY_SIZE(client); i++) {
3301 struct i915_request *rq;
3303 rq = create_gpr_client(engine, global,
3304 NUM_GPR * i * sizeof(u32));
3313 /* Continuously preempt the set of 3 running contexts */
3314 for (i = 1; i <= NUM_GPR; i++) {
3315 err = preempt_user(engine, global, i);
3320 if (READ_ONCE(result[0]) != NUM_GPR) {
3321 pr_err("%s: Failed to release semaphore\n",
3327 for (i = 0; i < ARRAY_SIZE(client); i++) {
3330 if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3335 for (gpr = 1; gpr < NUM_GPR; gpr++) {
3336 if (result[NUM_GPR * i + gpr] != 1) {
3337 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3339 i, gpr, result[NUM_GPR * i + gpr]);
3347 for (i = 0; i < ARRAY_SIZE(client); i++) {
3351 i915_request_put(client[i]);
3354 /* Flush the semaphores on error */
3355 smp_store_mb(result[0], -1);
3356 if (igt_live_test_end(&t))
3362 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3366 static int live_preempt_timeout(void *arg)
3368 struct intel_gt *gt = arg;
3369 struct i915_gem_context *ctx_hi, *ctx_lo;
3370 struct igt_spinner spin_lo;
3371 struct intel_engine_cs *engine;
3372 enum intel_engine_id id;
3376 * Check that we force preemption to occur by cancelling the previous
3377 * context if it refuses to yield the GPU.
3379 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
3382 if (!intel_has_reset_engine(gt))
3385 if (igt_spinner_init(&spin_lo, gt))
3388 ctx_hi = kernel_context(gt->i915, NULL);
3391 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
3393 ctx_lo = kernel_context(gt->i915, NULL);
3396 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
3398 for_each_engine(engine, gt, id) {
3399 unsigned long saved_timeout;
3400 struct i915_request *rq;
3402 if (!intel_engine_has_preemption(engine))
3405 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3406 MI_NOOP); /* preemption disabled */
3412 i915_request_add(rq);
3413 if (!igt_wait_for_spinner(&spin_lo, rq)) {
3414 intel_gt_set_wedged(gt);
3419 rq = igt_request_alloc(ctx_hi, engine);
3421 igt_spinner_end(&spin_lo);
3426 /* Flush the previous CS ack before changing timeouts */
3427 while (READ_ONCE(engine->execlists.pending[0]))
3430 saved_timeout = engine->props.preempt_timeout_ms;
3431 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
3433 i915_request_get(rq);
3434 i915_request_add(rq);
3436 intel_engine_flush_submission(engine);
3437 engine->props.preempt_timeout_ms = saved_timeout;
3439 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3440 intel_gt_set_wedged(gt);
3441 i915_request_put(rq);
3446 igt_spinner_end(&spin_lo);
3447 i915_request_put(rq);
3452 kernel_context_close(ctx_lo);
3454 kernel_context_close(ctx_hi);
3456 igt_spinner_fini(&spin_lo);
3460 static int random_range(struct rnd_state *rnd, int min, int max)
3462 return i915_prandom_u32_max_state(max - min, rnd) + min;
3465 static int random_priority(struct rnd_state *rnd)
3467 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3470 struct preempt_smoke {
3471 struct intel_gt *gt;
3472 struct i915_gem_context **contexts;
3473 struct intel_engine_cs *engine;
3474 struct drm_i915_gem_object *batch;
3475 unsigned int ncontext;
3476 struct rnd_state prng;
3477 unsigned long count;
3480 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3482 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3486 static int smoke_submit(struct preempt_smoke *smoke,
3487 struct i915_gem_context *ctx, int prio,
3488 struct drm_i915_gem_object *batch)
3490 struct i915_request *rq;
3491 struct i915_vma *vma = NULL;
3495 struct i915_address_space *vm;
3497 vm = i915_gem_context_get_eb_vm(ctx);
3498 vma = i915_vma_instance(batch, vm, NULL);
3501 return PTR_ERR(vma);
3503 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3508 ctx->sched.priority = prio;
3510 rq = igt_request_alloc(ctx, smoke->engine);
3518 err = i915_request_await_object(rq, vma->obj, false);
3520 err = i915_vma_move_to_active(vma, rq, 0);
3522 err = rq->engine->emit_bb_start(rq,
3525 i915_vma_unlock(vma);
3528 i915_request_add(rq);
3532 i915_vma_unpin(vma);
3537 static int smoke_crescendo_thread(void *arg)
3539 struct preempt_smoke *smoke = arg;
3540 IGT_TIMEOUT(end_time);
3541 unsigned long count;
3545 struct i915_gem_context *ctx = smoke_context(smoke);
3548 err = smoke_submit(smoke,
3549 ctx, count % I915_PRIORITY_MAX,
3555 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3557 smoke->count = count;
3561 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3562 #define BATCH BIT(0)
3564 struct task_struct *tsk[I915_NUM_ENGINES] = {};
3565 struct preempt_smoke *arg;
3566 struct intel_engine_cs *engine;
3567 enum intel_engine_id id;
3568 unsigned long count;
3571 arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL);
3575 for_each_engine(engine, smoke->gt, id) {
3577 arg[id].engine = engine;
3578 if (!(flags & BATCH))
3579 arg[id].batch = NULL;
3582 tsk[id] = kthread_run(smoke_crescendo_thread, arg,
3583 "igt/smoke:%d", id);
3584 if (IS_ERR(tsk[id])) {
3585 err = PTR_ERR(tsk[id]);
3588 get_task_struct(tsk[id]);
3591 yield(); /* start all threads before we kthread_stop() */
3594 for_each_engine(engine, smoke->gt, id) {
3597 if (IS_ERR_OR_NULL(tsk[id]))
3600 status = kthread_stop(tsk[id]);
3604 count += arg[id].count;
3606 put_task_struct(tsk[id]);
3609 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3610 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3616 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3618 enum intel_engine_id id;
3619 IGT_TIMEOUT(end_time);
3620 unsigned long count;
3624 for_each_engine(smoke->engine, smoke->gt, id) {
3625 struct i915_gem_context *ctx = smoke_context(smoke);
3628 err = smoke_submit(smoke,
3629 ctx, random_priority(&smoke->prng),
3630 flags & BATCH ? smoke->batch : NULL);
3636 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3638 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3639 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3643 static int live_preempt_smoke(void *arg)
3645 struct preempt_smoke smoke = {
3647 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3650 const unsigned int phase[] = { 0, BATCH };
3651 struct igt_live_test t;
3656 smoke.contexts = kmalloc_array(smoke.ncontext,
3657 sizeof(*smoke.contexts),
3659 if (!smoke.contexts)
3663 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3664 if (IS_ERR(smoke.batch)) {
3665 err = PTR_ERR(smoke.batch);
3669 cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
3674 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3675 cs[n] = MI_ARB_CHECK;
3676 cs[n] = MI_BATCH_BUFFER_END;
3677 i915_gem_object_flush_map(smoke.batch);
3678 i915_gem_object_unpin_map(smoke.batch);
3680 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3685 for (n = 0; n < smoke.ncontext; n++) {
3686 smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL);
3687 if (!smoke.contexts[n])
3691 for (n = 0; n < ARRAY_SIZE(phase); n++) {
3692 err = smoke_crescendo(&smoke, phase[n]);
3696 err = smoke_random(&smoke, phase[n]);
3702 if (igt_live_test_end(&t))
3705 for (n = 0; n < smoke.ncontext; n++) {
3706 if (!smoke.contexts[n])
3708 kernel_context_close(smoke.contexts[n]);
3712 i915_gem_object_put(smoke.batch);
3714 kfree(smoke.contexts);
3719 static int nop_virtual_engine(struct intel_gt *gt,
3720 struct intel_engine_cs **siblings,
3721 unsigned int nsibling,
3724 #define CHAIN BIT(0)
3726 IGT_TIMEOUT(end_time);
3727 struct i915_request *request[16] = {};
3728 struct intel_context *ve[16];
3729 unsigned long n, prime, nc;
3730 struct igt_live_test t;
3731 ktime_t times[2] = {};
3734 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3736 for (n = 0; n < nctx; n++) {
3737 ve[n] = intel_engine_create_virtual(siblings, nsibling, 0);
3738 if (IS_ERR(ve[n])) {
3739 err = PTR_ERR(ve[n]);
3744 err = intel_context_pin(ve[n]);
3746 intel_context_put(ve[n]);
3752 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3756 for_each_prime_number_from(prime, 1, 8192) {
3757 times[1] = ktime_get_raw();
3759 if (flags & CHAIN) {
3760 for (nc = 0; nc < nctx; nc++) {
3761 for (n = 0; n < prime; n++) {
3762 struct i915_request *rq;
3764 rq = i915_request_create(ve[nc]);
3771 i915_request_put(request[nc]);
3772 request[nc] = i915_request_get(rq);
3773 i915_request_add(rq);
3777 for (n = 0; n < prime; n++) {
3778 for (nc = 0; nc < nctx; nc++) {
3779 struct i915_request *rq;
3781 rq = i915_request_create(ve[nc]);
3788 i915_request_put(request[nc]);
3789 request[nc] = i915_request_get(rq);
3790 i915_request_add(rq);
3795 for (nc = 0; nc < nctx; nc++) {
3796 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3797 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3798 __func__, ve[0]->engine->name,
3799 request[nc]->fence.context,
3800 request[nc]->fence.seqno);
3802 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3803 __func__, ve[0]->engine->name,
3804 request[nc]->fence.context,
3805 request[nc]->fence.seqno);
3807 intel_gt_set_wedged(gt);
3812 times[1] = ktime_sub(ktime_get_raw(), times[1]);
3814 times[0] = times[1];
3816 for (nc = 0; nc < nctx; nc++) {
3817 i915_request_put(request[nc]);
3821 if (__igt_timeout(end_time, NULL))
3825 err = igt_live_test_end(&t);
3829 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3830 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3831 prime, div64_u64(ktime_to_ns(times[1]), prime));
3834 if (igt_flush_test(gt->i915))
3837 for (nc = 0; nc < nctx; nc++) {
3838 i915_request_put(request[nc]);
3839 intel_context_unpin(ve[nc]);
3840 intel_context_put(ve[nc]);
3846 __select_siblings(struct intel_gt *gt,
3848 struct intel_engine_cs **siblings,
3849 bool (*filter)(const struct intel_engine_cs *))
3854 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3855 if (!gt->engine_class[class][inst])
3858 if (filter && !filter(gt->engine_class[class][inst]))
3861 siblings[n++] = gt->engine_class[class][inst];
3868 select_siblings(struct intel_gt *gt,
3870 struct intel_engine_cs **siblings)
3872 return __select_siblings(gt, class, siblings, NULL);
3875 static int live_virtual_engine(void *arg)
3877 struct intel_gt *gt = arg;
3878 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3879 struct intel_engine_cs *engine;
3880 enum intel_engine_id id;
3884 if (intel_uc_uses_guc_submission(>->uc))
3887 for_each_engine(engine, gt, id) {
3888 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3890 pr_err("Failed to wrap engine %s: err=%d\n",
3896 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3899 nsibling = select_siblings(gt, class, siblings);
3903 for (n = 1; n <= nsibling + 1; n++) {
3904 err = nop_virtual_engine(gt, siblings, nsibling,
3910 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3918 static int mask_virtual_engine(struct intel_gt *gt,
3919 struct intel_engine_cs **siblings,
3920 unsigned int nsibling)
3922 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3923 struct intel_context *ve;
3924 struct igt_live_test t;
3929 * Check that by setting the execution mask on a request, we can
3930 * restrict it to our desired engine within the virtual engine.
3933 ve = intel_engine_create_virtual(siblings, nsibling, 0);
3939 err = intel_context_pin(ve);
3943 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3947 for (n = 0; n < nsibling; n++) {
3948 request[n] = i915_request_create(ve);
3949 if (IS_ERR(request[n])) {
3950 err = PTR_ERR(request[n]);
3955 /* Reverse order as it's more likely to be unnatural */
3956 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3958 i915_request_get(request[n]);
3959 i915_request_add(request[n]);
3962 for (n = 0; n < nsibling; n++) {
3963 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3964 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3965 __func__, ve->engine->name,
3966 request[n]->fence.context,
3967 request[n]->fence.seqno);
3969 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3970 __func__, ve->engine->name,
3971 request[n]->fence.context,
3972 request[n]->fence.seqno);
3974 intel_gt_set_wedged(gt);
3979 if (request[n]->engine != siblings[nsibling - n - 1]) {
3980 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3981 request[n]->engine->name,
3982 siblings[nsibling - n - 1]->name);
3988 err = igt_live_test_end(&t);
3990 if (igt_flush_test(gt->i915))
3993 for (n = 0; n < nsibling; n++)
3994 i915_request_put(request[n]);
3997 intel_context_unpin(ve);
3999 intel_context_put(ve);
4004 static int live_virtual_mask(void *arg)
4006 struct intel_gt *gt = arg;
4007 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4011 if (intel_uc_uses_guc_submission(>->uc))
4014 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4015 unsigned int nsibling;
4017 nsibling = select_siblings(gt, class, siblings);
4021 err = mask_virtual_engine(gt, siblings, nsibling);
4029 static int slicein_virtual_engine(struct intel_gt *gt,
4030 struct intel_engine_cs **siblings,
4031 unsigned int nsibling)
4033 const long timeout = slice_timeout(siblings[0]);
4034 struct intel_context *ce;
4035 struct i915_request *rq;
4036 struct igt_spinner spin;
4041 * Virtual requests must take part in timeslicing on the target engines.
4044 if (igt_spinner_init(&spin, gt))
4047 for (n = 0; n < nsibling; n++) {
4048 ce = intel_context_create(siblings[n]);
4054 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4055 intel_context_put(ce);
4061 i915_request_add(rq);
4064 ce = intel_engine_create_virtual(siblings, nsibling, 0);
4070 rq = intel_context_create_request(ce);
4071 intel_context_put(ce);
4077 i915_request_get(rq);
4078 i915_request_add(rq);
4079 if (i915_request_wait(rq, 0, timeout) < 0) {
4080 GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",
4081 __func__, rq->engine->name);
4083 intel_gt_set_wedged(gt);
4086 i915_request_put(rq);
4089 igt_spinner_end(&spin);
4090 if (igt_flush_test(gt->i915))
4092 igt_spinner_fini(&spin);
4096 static int sliceout_virtual_engine(struct intel_gt *gt,
4097 struct intel_engine_cs **siblings,
4098 unsigned int nsibling)
4100 const long timeout = slice_timeout(siblings[0]);
4101 struct intel_context *ce;
4102 struct i915_request *rq;
4103 struct igt_spinner spin;
4108 * Virtual requests must allow others a fair timeslice.
4111 if (igt_spinner_init(&spin, gt))
4114 /* XXX We do not handle oversubscription and fairness with normal rq */
4115 for (n = 0; n < nsibling; n++) {
4116 ce = intel_engine_create_virtual(siblings, nsibling, 0);
4122 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4123 intel_context_put(ce);
4129 i915_request_add(rq);
4132 for (n = 0; !err && n < nsibling; n++) {
4133 ce = intel_context_create(siblings[n]);
4139 rq = intel_context_create_request(ce);
4140 intel_context_put(ce);
4146 i915_request_get(rq);
4147 i915_request_add(rq);
4148 if (i915_request_wait(rq, 0, timeout) < 0) {
4149 GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",
4150 __func__, siblings[n]->name);
4152 intel_gt_set_wedged(gt);
4155 i915_request_put(rq);
4159 igt_spinner_end(&spin);
4160 if (igt_flush_test(gt->i915))
4162 igt_spinner_fini(&spin);
4166 static int live_virtual_slice(void *arg)
4168 struct intel_gt *gt = arg;
4169 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4173 if (intel_uc_uses_guc_submission(>->uc))
4176 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4177 unsigned int nsibling;
4179 nsibling = __select_siblings(gt, class, siblings,
4180 intel_engine_has_timeslices);
4184 err = slicein_virtual_engine(gt, siblings, nsibling);
4188 err = sliceout_virtual_engine(gt, siblings, nsibling);
4196 static int preserved_virtual_engine(struct intel_gt *gt,
4197 struct intel_engine_cs **siblings,
4198 unsigned int nsibling)
4200 struct i915_request *last = NULL;
4201 struct intel_context *ve;
4202 struct i915_vma *scratch;
4203 struct igt_live_test t;
4209 __vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
4211 if (IS_ERR(scratch))
4212 return PTR_ERR(scratch);
4214 err = i915_vma_sync(scratch);
4218 ve = intel_engine_create_virtual(siblings, nsibling, 0);
4224 err = intel_context_pin(ve);
4228 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
4232 for (n = 0; n < NUM_GPR_DW; n++) {
4233 struct intel_engine_cs *engine = siblings[n % nsibling];
4234 struct i915_request *rq;
4236 rq = i915_request_create(ve);
4242 i915_request_put(last);
4243 last = i915_request_get(rq);
4245 cs = intel_ring_begin(rq, 8);
4247 i915_request_add(rq);
4252 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4253 *cs++ = CS_GPR(engine, n);
4254 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4257 *cs++ = MI_LOAD_REGISTER_IMM(1);
4258 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
4262 intel_ring_advance(rq, cs);
4264 /* Restrict this request to run on a particular engine */
4265 rq->execution_mask = engine->mask;
4266 i915_request_add(rq);
4269 if (i915_request_wait(last, 0, HZ / 5) < 0) {
4274 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
4280 for (n = 0; n < NUM_GPR_DW; n++) {
4282 pr_err("Incorrect value[%d] found for GPR[%d]\n",
4289 i915_gem_object_unpin_map(scratch->obj);
4292 if (igt_live_test_end(&t))
4294 i915_request_put(last);
4296 intel_context_unpin(ve);
4298 intel_context_put(ve);
4300 i915_vma_unpin_and_release(&scratch, 0);
4304 static int live_virtual_preserved(void *arg)
4306 struct intel_gt *gt = arg;
4307 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4311 * Check that the context image retains non-privileged (user) registers
4312 * from one engine to the next. For this we check that the CS_GPR
4316 if (intel_uc_uses_guc_submission(>->uc))
4319 /* As we use CS_GPR we cannot run before they existed on all engines. */
4320 if (GRAPHICS_VER(gt->i915) < 9)
4323 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4326 nsibling = select_siblings(gt, class, siblings);
4330 err = preserved_virtual_engine(gt, siblings, nsibling);
4338 static int reset_virtual_engine(struct intel_gt *gt,
4339 struct intel_engine_cs **siblings,
4340 unsigned int nsibling)
4342 struct intel_engine_cs *engine;
4343 struct intel_context *ve;
4344 struct igt_spinner spin;
4345 struct i915_request *rq;
4350 * In order to support offline error capture for fast preempt reset,
4351 * we need to decouple the guilty request and ensure that it and its
4352 * descendents are not executed while the capture is in progress.
4355 if (igt_spinner_init(&spin, gt))
4358 ve = intel_engine_create_virtual(siblings, nsibling, 0);
4364 for (n = 0; n < nsibling; n++)
4365 st_engine_heartbeat_disable(siblings[n]);
4367 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4372 i915_request_add(rq);
4374 if (!igt_wait_for_spinner(&spin, rq)) {
4375 intel_gt_set_wedged(gt);
4380 engine = rq->engine;
4381 GEM_BUG_ON(engine == ve->engine);
4383 /* Take ownership of the reset and tasklet */
4384 err = engine_lock_reset_tasklet(engine);
4388 engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);
4389 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4391 /* Fake a preemption event; failed of course */
4392 spin_lock_irq(&engine->sched_engine->lock);
4393 __unwind_incomplete_requests(engine);
4394 spin_unlock_irq(&engine->sched_engine->lock);
4395 GEM_BUG_ON(rq->engine != engine);
4397 /* Reset the engine while keeping our active request on hold */
4398 execlists_hold(engine, rq);
4399 GEM_BUG_ON(!i915_request_on_hold(rq));
4401 __intel_engine_reset_bh(engine, NULL);
4402 GEM_BUG_ON(rq->fence.error != -EIO);
4404 /* Release our grasp on the engine, letting CS flow again */
4405 engine_unlock_reset_tasklet(engine);
4407 /* Check that we do not resubmit the held request */
4408 i915_request_get(rq);
4409 if (!i915_request_wait(rq, 0, HZ / 5)) {
4410 pr_err("%s: on hold request completed!\n",
4412 intel_gt_set_wedged(gt);
4416 GEM_BUG_ON(!i915_request_on_hold(rq));
4418 /* But is resubmitted on release */
4419 execlists_unhold(engine, rq);
4420 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4421 pr_err("%s: held request did not complete!\n",
4423 intel_gt_set_wedged(gt);
4428 i915_request_put(rq);
4430 for (n = 0; n < nsibling; n++)
4431 st_engine_heartbeat_enable(siblings[n]);
4433 intel_context_put(ve);
4435 igt_spinner_fini(&spin);
4439 static int live_virtual_reset(void *arg)
4441 struct intel_gt *gt = arg;
4442 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4446 * Check that we handle a reset event within a virtual engine.
4447 * Only the physical engine is reset, but we have to check the flow
4448 * of the virtual requests around the reset, and make sure it is not
4452 if (intel_uc_uses_guc_submission(>->uc))
4455 if (!intel_has_reset_engine(gt))
4458 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4461 nsibling = select_siblings(gt, class, siblings);
4465 err = reset_virtual_engine(gt, siblings, nsibling);
4473 int intel_execlists_live_selftests(struct drm_i915_private *i915)
4475 static const struct i915_subtest tests[] = {
4476 SUBTEST(live_sanitycheck),
4477 SUBTEST(live_unlite_switch),
4478 SUBTEST(live_unlite_preempt),
4479 SUBTEST(live_unlite_ring),
4480 SUBTEST(live_pin_rewind),
4481 SUBTEST(live_hold_reset),
4482 SUBTEST(live_error_interrupt),
4483 SUBTEST(live_timeslice_preempt),
4484 SUBTEST(live_timeslice_rewind),
4485 SUBTEST(live_timeslice_queue),
4486 SUBTEST(live_timeslice_nopreempt),
4487 SUBTEST(live_busywait_preempt),
4488 SUBTEST(live_preempt),
4489 SUBTEST(live_late_preempt),
4490 SUBTEST(live_nopreempt),
4491 SUBTEST(live_preempt_cancel),
4492 SUBTEST(live_suppress_self_preempt),
4493 SUBTEST(live_chain_preempt),
4494 SUBTEST(live_preempt_ring),
4495 SUBTEST(live_preempt_gang),
4496 SUBTEST(live_preempt_timeout),
4497 SUBTEST(live_preempt_user),
4498 SUBTEST(live_preempt_smoke),
4499 SUBTEST(live_virtual_engine),
4500 SUBTEST(live_virtual_mask),
4501 SUBTEST(live_virtual_preserved),
4502 SUBTEST(live_virtual_slice),
4503 SUBTEST(live_virtual_reset),
4506 if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP)
4509 if (intel_gt_is_wedged(to_gt(i915)))
4512 return intel_gt_live_subtests(tests, to_gt(i915));