drivers/gpu/drm/i915/gt/selftest_lrc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include <linux/prime_numbers.h>
   7
   8 #include "i915_selftest.h"
   9 #include "intel_engine_heartbeat.h"
  10 #include "intel_engine_pm.h"
  11 #include "intel_reset.h"
  12 #include "intel_ring.h"
  13 #include "selftest_engine_heartbeat.h"
  14 #include "selftests/i915_random.h"
  15 #include "selftests/igt_flush_test.h"
  16 #include "selftests/igt_live_test.h"
  17 #include "selftests/igt_spinner.h"
  18 #include "selftests/lib_sw_fence.h"
  19 #include "shmem_utils.h"
  20
  21 #include "gem/selftests/igt_gem_utils.h"
  22 #include "gem/selftests/mock_context.h"
  23
  24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
  25 #define NUM_GPR 16
  26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
  27
  28 static struct i915_vma *create_scratch(struct intel_gt *gt)
  29 {
  30         return __vm_create_scratch_for_read(&gt->ggtt->vm, PAGE_SIZE);
  31 }
  32
  33 static bool is_active(struct i915_request *rq)
  34 {
  35         if (i915_request_is_active(rq))
  36                 return true;
  37
  38         if (i915_request_on_hold(rq))
  39                 return true;
  40
  41         if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
  42                 return true;
  43
  44         return false;
  45 }
  46
  47 static int wait_for_submit(struct intel_engine_cs *engine,
  48                            struct i915_request *rq,
  49                            unsigned long timeout)
  50 {
  51         /* Ignore our own attempts to suppress excess tasklets */
  52         tasklet_hi_schedule(&engine->execlists.tasklet);
  53
  54         timeout += jiffies;
  55         do {
  56                 bool done = time_after(jiffies, timeout);
  57
  58                 if (i915_request_completed(rq)) /* that was quick! */
  59                         return 0;
  60
  61                 /* Wait until the HW has acknowleged the submission (or err) */
  62                 intel_engine_flush_submission(engine);
  63                 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
  64                         return 0;
  65
  66                 if (done)
  67                         return -ETIME;
  68
  69                 cond_resched();
  70         } while (1);
  71 }
  72
  73 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
  74 {
  75         const u32 offset =
  76                 i915_ggtt_offset(ce->engine->status_page.vma) +
  77                 offset_in_page(slot);
  78         struct i915_request *rq;
  79         u32 *cs;
  80
  81         rq = intel_context_create_request(ce);
  82         if (IS_ERR(rq))
  83                 return PTR_ERR(rq);
  84
  85         cs = intel_ring_begin(rq, 4);
  86         if (IS_ERR(cs)) {
  87                 i915_request_add(rq);
  88                 return PTR_ERR(cs);
  89         }
  90
  91         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  92         *cs++ = offset;
  93         *cs++ = 0;
  94         *cs++ = 1;
  95
  96         intel_ring_advance(rq, cs);
  97
  98         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
  99         i915_request_add(rq);
 100         return 0;
 101 }
 102
 103 static int context_flush(struct intel_context *ce, long timeout)
 104 {
 105         struct i915_request *rq;
 106         struct dma_fence *fence;
 107         int err = 0;
 108
 109         rq = intel_engine_create_kernel_request(ce->engine);
 110         if (IS_ERR(rq))
 111                 return PTR_ERR(rq);
 112
 113         fence = i915_active_fence_get(&ce->timeline->last_request);
 114         if (fence) {
 115                 i915_request_await_dma_fence(rq, fence);
 116                 dma_fence_put(fence);
 117         }
 118
 119         rq = i915_request_get(rq);
 120         i915_request_add(rq);
 121         if (i915_request_wait(rq, 0, timeout) < 0)
 122                 err = -ETIME;
 123         i915_request_put(rq);
 124
 125         rmb(); /* We know the request is written, make sure all state is too! */
 126         return err;
 127 }
 128
 129 static int live_lrc_layout(void *arg)
 130 {
 131         struct intel_gt *gt = arg;
 132         struct intel_engine_cs *engine;
 133         enum intel_engine_id id;
 134         u32 *lrc;
 135         int err;
 136
 137         /*
 138          * Check the registers offsets we use to create the initial reg state
 139          * match the layout saved by HW.
 140          */
 141
 142         lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
 143         if (!lrc)
 144                 return -ENOMEM;
 145         GEM_BUG_ON(offset_in_page(lrc));
 146
 147         err = 0;
 148         for_each_engine(engine, gt, id) {
 149                 u32 *hw;
 150                 int dw;
 151
 152                 if (!engine->default_state)
 153                         continue;
 154
 155                 hw = shmem_pin_map(engine->default_state);
 156                 if (IS_ERR(hw)) {
 157                         err = PTR_ERR(hw);
 158                         break;
 159                 }
 160                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 161
 162                 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
 163                                 engine->kernel_context, engine, true);
 164
 165                 dw = 0;
 166                 do {
 167                         u32 lri = hw[dw];
 168
 169                         if (lri == 0) {
 170                                 dw++;
 171                                 continue;
 172                         }
 173
 174                         if (lrc[dw] == 0) {
 175                                 pr_debug("%s: skipped instruction %x at dword %d\n",
 176                                          engine->name, lri, dw);
 177                                 dw++;
 178                                 continue;
 179                         }
 180
 181                         if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 182                                 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
 183                                        engine->name, dw, lri);
 184                                 err = -EINVAL;
 185                                 break;
 186                         }
 187
 188                         if (lrc[dw] != lri) {
 189                                 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
 190                                        engine->name, dw, lri, lrc[dw]);
 191                                 err = -EINVAL;
 192                                 break;
 193                         }
 194
 195                         lri &= 0x7f;
 196                         lri++;
 197                         dw++;
 198
 199                         while (lri) {
 200                                 if (hw[dw] != lrc[dw]) {
 201                                         pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
 202                                                engine->name, dw, hw[dw], lrc[dw]);
 203                                         err = -EINVAL;
 204                                         break;
 205                                 }
 206
 207                                 /*
 208                                  * Skip over the actual register value as we
 209                                  * expect that to differ.
 210                                  */
 211                                 dw += 2;
 212                                 lri -= 2;
 213                         }
 214                 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 215
 216                 if (err) {
 217                         pr_info("%s: HW register image:\n", engine->name);
 218                         igt_hexdump(hw, PAGE_SIZE);
 219
 220                         pr_info("%s: SW register image:\n", engine->name);
 221                         igt_hexdump(lrc, PAGE_SIZE);
 222                 }
 223
 224                 shmem_unpin_map(engine->default_state, hw);
 225                 if (err)
 226                         break;
 227         }
 228
 229         free_page((unsigned long)lrc);
 230         return err;
 231 }
 232
 233 static int find_offset(const u32 *lri, u32 offset)
 234 {
 235         int i;
 236
 237         for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
 238                 if (lri[i] == offset)
 239                         return i;
 240
 241         return -1;
 242 }
 243
 244 static int live_lrc_fixed(void *arg)
 245 {
 246         struct intel_gt *gt = arg;
 247         struct intel_engine_cs *engine;
 248         enum intel_engine_id id;
 249         int err = 0;
 250
 251         /*
 252          * Check the assumed register offsets match the actual locations in
 253          * the context image.
 254          */
 255
 256         for_each_engine(engine, gt, id) {
 257                 const struct {
 258                         u32 reg;
 259                         u32 offset;
 260                         const char *name;
 261                 } tbl[] = {
 262                         {
 263                                 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
 264                                 CTX_RING_START - 1,
 265                                 "RING_START"
 266                         },
 267                         {
 268                                 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
 269                                 CTX_RING_CTL - 1,
 270                                 "RING_CTL"
 271                         },
 272                         {
 273                                 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
 274                                 CTX_RING_HEAD - 1,
 275                                 "RING_HEAD"
 276                         },
 277                         {
 278                                 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
 279                                 CTX_RING_TAIL - 1,
 280                                 "RING_TAIL"
 281                         },
 282                         {
 283                                 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
 284                                 lrc_ring_mi_mode(engine),
 285                                 "RING_MI_MODE"
 286                         },
 287                         {
 288                                 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
 289                                 CTX_BB_STATE - 1,
 290                                 "BB_STATE"
 291                         },
 292                         {
 293                                 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
 294                                 lrc_ring_wa_bb_per_ctx(engine),
 295                                 "RING_BB_PER_CTX_PTR"
 296                         },
 297                         {
 298                                 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
 299                                 lrc_ring_indirect_ptr(engine),
 300                                 "RING_INDIRECT_CTX_PTR"
 301                         },
 302                         {
 303                                 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
 304                                 lrc_ring_indirect_offset(engine),
 305                                 "RING_INDIRECT_CTX_OFFSET"
 306                         },
 307                         {
 308                                 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
 309                                 CTX_TIMESTAMP - 1,
 310                                 "RING_CTX_TIMESTAMP"
 311                         },
 312                         {
 313                                 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
 314                                 lrc_ring_gpr0(engine),
 315                                 "RING_CS_GPR0"
 316                         },
 317                         {
 318                                 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
 319                                 lrc_ring_cmd_buf_cctl(engine),
 320                                 "RING_CMD_BUF_CCTL"
 321                         },
 322                         { },
 323                 }, *t;
 324                 u32 *hw;
 325
 326                 if (!engine->default_state)
 327                         continue;
 328
 329                 hw = shmem_pin_map(engine->default_state);
 330                 if (IS_ERR(hw)) {
 331                         err = PTR_ERR(hw);
 332                         break;
 333                 }
 334                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 335
 336                 for (t = tbl; t->name; t++) {
 337                         int dw = find_offset(hw, t->reg);
 338
 339                         if (dw != t->offset) {
 340                                 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
 341                                        engine->name,
 342                                        t->name,
 343                                        t->reg,
 344                                        dw,
 345                                        t->offset);
 346                                 err = -EINVAL;
 347                         }
 348                 }
 349
 350                 shmem_unpin_map(engine->default_state, hw);
 351         }
 352
 353         return err;
 354 }
 355
 356 static int __live_lrc_state(struct intel_engine_cs *engine,
 357                             struct i915_vma *scratch)
 358 {
 359         struct intel_context *ce;
 360         struct i915_request *rq;
 361         struct i915_gem_ww_ctx ww;
 362         enum {
 363                 RING_START_IDX = 0,
 364                 RING_TAIL_IDX,
 365                 MAX_IDX
 366         };
 367         u32 expected[MAX_IDX];
 368         u32 *cs;
 369         int err;
 370         int n;
 371
 372         ce = intel_context_create(engine);
 373         if (IS_ERR(ce))
 374                 return PTR_ERR(ce);
 375
 376         i915_gem_ww_ctx_init(&ww, false);
 377 retry:
 378         err = i915_gem_object_lock(scratch->obj, &ww);
 379         if (!err)
 380                 err = intel_context_pin_ww(ce, &ww);
 381         if (err)
 382                 goto err_put;
 383
 384         rq = i915_request_create(ce);
 385         if (IS_ERR(rq)) {
 386                 err = PTR_ERR(rq);
 387                 goto err_unpin;
 388         }
 389
 390         cs = intel_ring_begin(rq, 4 * MAX_IDX);
 391         if (IS_ERR(cs)) {
 392                 err = PTR_ERR(cs);
 393                 i915_request_add(rq);
 394                 goto err_unpin;
 395         }
 396
 397         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 398         *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
 399         *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
 400         *cs++ = 0;
 401
 402         expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
 403
 404         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 405         *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
 406         *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
 407         *cs++ = 0;
 408
 409         err = i915_request_await_object(rq, scratch->obj, true);
 410         if (!err)
 411                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 412
 413         i915_request_get(rq);
 414         i915_request_add(rq);
 415         if (err)
 416                 goto err_rq;
 417
 418         intel_engine_flush_submission(engine);
 419         expected[RING_TAIL_IDX] = ce->ring->tail;
 420
 421         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 422                 err = -ETIME;
 423                 goto err_rq;
 424         }
 425
 426         cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 427         if (IS_ERR(cs)) {
 428                 err = PTR_ERR(cs);
 429                 goto err_rq;
 430         }
 431
 432         for (n = 0; n < MAX_IDX; n++) {
 433                 if (cs[n] != expected[n]) {
 434                         pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
 435                                engine->name, n, cs[n], expected[n]);
 436                         err = -EINVAL;
 437                         break;
 438                 }
 439         }
 440
 441         i915_gem_object_unpin_map(scratch->obj);
 442
 443 err_rq:
 444         i915_request_put(rq);
 445 err_unpin:
 446         intel_context_unpin(ce);
 447 err_put:
 448         if (err == -EDEADLK) {
 449                 err = i915_gem_ww_ctx_backoff(&ww);
 450                 if (!err)
 451                         goto retry;
 452         }
 453         i915_gem_ww_ctx_fini(&ww);
 454         intel_context_put(ce);
 455         return err;
 456 }
 457
 458 static int live_lrc_state(void *arg)
 459 {
 460         struct intel_gt *gt = arg;
 461         struct intel_engine_cs *engine;
 462         struct i915_vma *scratch;
 463         enum intel_engine_id id;
 464         int err = 0;
 465
 466         /*
 467          * Check the live register state matches what we expect for this
 468          * intel_context.
 469          */
 470
 471         scratch = create_scratch(gt);
 472         if (IS_ERR(scratch))
 473                 return PTR_ERR(scratch);
 474
 475         for_each_engine(engine, gt, id) {
 476                 err = __live_lrc_state(engine, scratch);
 477                 if (err)
 478                         break;
 479         }
 480
 481         if (igt_flush_test(gt->i915))
 482                 err = -EIO;
 483
 484         i915_vma_unpin_and_release(&scratch, 0);
 485         return err;
 486 }
 487
 488 static int gpr_make_dirty(struct intel_context *ce)
 489 {
 490         struct i915_request *rq;
 491         u32 *cs;
 492         int n;
 493
 494         rq = intel_context_create_request(ce);
 495         if (IS_ERR(rq))
 496                 return PTR_ERR(rq);
 497
 498         cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
 499         if (IS_ERR(cs)) {
 500                 i915_request_add(rq);
 501                 return PTR_ERR(cs);
 502         }
 503
 504         *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
 505         for (n = 0; n < NUM_GPR_DW; n++) {
 506                 *cs++ = CS_GPR(ce->engine, n);
 507                 *cs++ = STACK_MAGIC;
 508         }
 509         *cs++ = MI_NOOP;
 510
 511         intel_ring_advance(rq, cs);
 512
 513         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
 514         i915_request_add(rq);
 515
 516         return 0;
 517 }
 518
 519 static struct i915_request *
 520 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
 521 {
 522         const u32 offset =
 523                 i915_ggtt_offset(ce->engine->status_page.vma) +
 524                 offset_in_page(slot);
 525         struct i915_request *rq;
 526         u32 *cs;
 527         int err;
 528         int n;
 529
 530         rq = intel_context_create_request(ce);
 531         if (IS_ERR(rq))
 532                 return rq;
 533
 534         cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
 535         if (IS_ERR(cs)) {
 536                 i915_request_add(rq);
 537                 return ERR_CAST(cs);
 538         }
 539
 540         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 541         *cs++ = MI_NOOP;
 542
 543         *cs++ = MI_SEMAPHORE_WAIT |
 544                 MI_SEMAPHORE_GLOBAL_GTT |
 545                 MI_SEMAPHORE_POLL |
 546                 MI_SEMAPHORE_SAD_NEQ_SDD;
 547         *cs++ = 0;
 548         *cs++ = offset;
 549         *cs++ = 0;
 550
 551         for (n = 0; n < NUM_GPR_DW; n++) {
 552                 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 553                 *cs++ = CS_GPR(ce->engine, n);
 554                 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
 555                 *cs++ = 0;
 556         }
 557
 558         i915_vma_lock(scratch);
 559         err = i915_request_await_object(rq, scratch->obj, true);
 560         if (!err)
 561                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 562         i915_vma_unlock(scratch);
 563
 564         i915_request_get(rq);
 565         i915_request_add(rq);
 566         if (err) {
 567                 i915_request_put(rq);
 568                 rq = ERR_PTR(err);
 569         }
 570
 571         return rq;
 572 }
 573
 574 static int __live_lrc_gpr(struct intel_engine_cs *engine,
 575                           struct i915_vma *scratch,
 576                           bool preempt)
 577 {
 578         u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
 579         struct intel_context *ce;
 580         struct i915_request *rq;
 581         u32 *cs;
 582         int err;
 583         int n;
 584
 585         if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
 586                 return 0; /* GPR only on rcs0 for gen8 */
 587
 588         err = gpr_make_dirty(engine->kernel_context);
 589         if (err)
 590                 return err;
 591
 592         ce = intel_context_create(engine);
 593         if (IS_ERR(ce))
 594                 return PTR_ERR(ce);
 595
 596         rq = __gpr_read(ce, scratch, slot);
 597         if (IS_ERR(rq)) {
 598                 err = PTR_ERR(rq);
 599                 goto err_put;
 600         }
 601
 602         err = wait_for_submit(engine, rq, HZ / 2);
 603         if (err)
 604                 goto err_rq;
 605
 606         if (preempt) {
 607                 err = gpr_make_dirty(engine->kernel_context);
 608                 if (err)
 609                         goto err_rq;
 610
 611                 err = emit_semaphore_signal(engine->kernel_context, slot);
 612                 if (err)
 613                         goto err_rq;
 614
 615                 err = wait_for_submit(engine, rq, HZ / 2);
 616                 if (err)
 617                         goto err_rq;
 618         } else {
 619                 slot[0] = 1;
 620                 wmb();
 621         }
 622
 623         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 624                 err = -ETIME;
 625                 goto err_rq;
 626         }
 627
 628         cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 629         if (IS_ERR(cs)) {
 630                 err = PTR_ERR(cs);
 631                 goto err_rq;
 632         }
 633
 634         for (n = 0; n < NUM_GPR_DW; n++) {
 635                 if (cs[n]) {
 636                         pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
 637                                engine->name,
 638                                n / 2, n & 1 ? "udw" : "ldw",
 639                                cs[n]);
 640                         err = -EINVAL;
 641                         break;
 642                 }
 643         }
 644
 645         i915_gem_object_unpin_map(scratch->obj);
 646
 647 err_rq:
 648         memset32(&slot[0], -1, 4);
 649         wmb();
 650         i915_request_put(rq);
 651 err_put:
 652         intel_context_put(ce);
 653         return err;
 654 }
 655
 656 static int live_lrc_gpr(void *arg)
 657 {
 658         struct intel_gt *gt = arg;
 659         struct intel_engine_cs *engine;
 660         struct i915_vma *scratch;
 661         enum intel_engine_id id;
 662         int err = 0;
 663
 664         /*
 665          * Check that GPR registers are cleared in new contexts as we need
 666          * to avoid leaking any information from previous contexts.
 667          */
 668
 669         scratch = create_scratch(gt);
 670         if (IS_ERR(scratch))
 671                 return PTR_ERR(scratch);
 672
 673         for_each_engine(engine, gt, id) {
 674                 st_engine_heartbeat_disable(engine);
 675
 676                 err = __live_lrc_gpr(engine, scratch, false);
 677                 if (err)
 678                         goto err;
 679
 680                 err = __live_lrc_gpr(engine, scratch, true);
 681                 if (err)
 682                         goto err;
 683
 684 err:
 685                 st_engine_heartbeat_enable(engine);
 686                 if (igt_flush_test(gt->i915))
 687                         err = -EIO;
 688                 if (err)
 689                         break;
 690         }
 691
 692         i915_vma_unpin_and_release(&scratch, 0);
 693         return err;
 694 }
 695
 696 static struct i915_request *
 697 create_timestamp(struct intel_context *ce, void *slot, int idx)
 698 {
 699         const u32 offset =
 700                 i915_ggtt_offset(ce->engine->status_page.vma) +
 701                 offset_in_page(slot);
 702         struct i915_request *rq;
 703         u32 *cs;
 704         int err;
 705
 706         rq = intel_context_create_request(ce);
 707         if (IS_ERR(rq))
 708                 return rq;
 709
 710         cs = intel_ring_begin(rq, 10);
 711         if (IS_ERR(cs)) {
 712                 err = PTR_ERR(cs);
 713                 goto err;
 714         }
 715
 716         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 717         *cs++ = MI_NOOP;
 718
 719         *cs++ = MI_SEMAPHORE_WAIT |
 720                 MI_SEMAPHORE_GLOBAL_GTT |
 721                 MI_SEMAPHORE_POLL |
 722                 MI_SEMAPHORE_SAD_NEQ_SDD;
 723         *cs++ = 0;
 724         *cs++ = offset;
 725         *cs++ = 0;
 726
 727         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 728         *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
 729         *cs++ = offset + idx * sizeof(u32);
 730         *cs++ = 0;
 731
 732         intel_ring_advance(rq, cs);
 733
 734         rq->sched.attr.priority = I915_PRIORITY_MASK;
 735         err = 0;
 736 err:
 737         i915_request_get(rq);
 738         i915_request_add(rq);
 739         if (err) {
 740                 i915_request_put(rq);
 741                 return ERR_PTR(err);
 742         }
 743
 744         return rq;
 745 }
 746
 747 struct lrc_timestamp {
 748         struct intel_engine_cs *engine;
 749         struct intel_context *ce[2];
 750         u32 poison;
 751 };
 752
 753 static bool timestamp_advanced(u32 start, u32 end)
 754 {
 755         return (s32)(end - start) > 0;
 756 }
 757
 758 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
 759 {
 760         u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
 761         struct i915_request *rq;
 762         u32 timestamp;
 763         int err = 0;
 764
 765         arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
 766         rq = create_timestamp(arg->ce[0], slot, 1);
 767         if (IS_ERR(rq))
 768                 return PTR_ERR(rq);
 769
 770         err = wait_for_submit(rq->engine, rq, HZ / 2);
 771         if (err)
 772                 goto err;
 773
 774         if (preempt) {
 775                 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
 776                 err = emit_semaphore_signal(arg->ce[1], slot);
 777                 if (err)
 778                         goto err;
 779         } else {
 780                 slot[0] = 1;
 781                 wmb();
 782         }
 783
 784         /* And wait for switch to kernel (to save our context to memory) */
 785         err = context_flush(arg->ce[0], HZ / 2);
 786         if (err)
 787                 goto err;
 788
 789         if (!timestamp_advanced(arg->poison, slot[1])) {
 790                 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
 791                        arg->engine->name, preempt ? "preempt" : "simple",
 792                        arg->poison, slot[1]);
 793                 err = -EINVAL;
 794         }
 795
 796         timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
 797         if (!timestamp_advanced(slot[1], timestamp)) {
 798                 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
 799                        arg->engine->name, preempt ? "preempt" : "simple",
 800                        slot[1], timestamp);
 801                 err = -EINVAL;
 802         }
 803
 804 err:
 805         memset32(slot, -1, 4);
 806         i915_request_put(rq);
 807         return err;
 808 }
 809
 810 static int live_lrc_timestamp(void *arg)
 811 {
 812         struct lrc_timestamp data = {};
 813         struct intel_gt *gt = arg;
 814         enum intel_engine_id id;
 815         const u32 poison[] = {
 816                 0,
 817                 S32_MAX,
 818                 (u32)S32_MAX + 1,
 819                 U32_MAX,
 820         };
 821
 822         /*
 823          * We want to verify that the timestamp is saved and restore across
 824          * context switches and is monotonic.
 825          *
 826          * So we do this with a little bit of LRC poisoning to check various
 827          * boundary conditions, and see what happens if we preempt the context
 828          * with a second request (carrying more poison into the timestamp).
 829          */
 830
 831         for_each_engine(data.engine, gt, id) {
 832                 int i, err = 0;
 833
 834                 st_engine_heartbeat_disable(data.engine);
 835
 836                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 837                         struct intel_context *tmp;
 838
 839                         tmp = intel_context_create(data.engine);
 840                         if (IS_ERR(tmp)) {
 841                                 err = PTR_ERR(tmp);
 842                                 goto err;
 843                         }
 844
 845                         err = intel_context_pin(tmp);
 846                         if (err) {
 847                                 intel_context_put(tmp);
 848                                 goto err;
 849                         }
 850
 851                         data.ce[i] = tmp;
 852                 }
 853
 854                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
 855                         data.poison = poison[i];
 856
 857                         err = __lrc_timestamp(&data, false);
 858                         if (err)
 859                                 break;
 860
 861                         err = __lrc_timestamp(&data, true);
 862                         if (err)
 863                                 break;
 864                 }
 865
 866 err:
 867                 st_engine_heartbeat_enable(data.engine);
 868                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 869                         if (!data.ce[i])
 870                                 break;
 871
 872                         intel_context_unpin(data.ce[i]);
 873                         intel_context_put(data.ce[i]);
 874                 }
 875
 876                 if (igt_flush_test(gt->i915))
 877                         err = -EIO;
 878                 if (err)
 879                         return err;
 880         }
 881
 882         return 0;
 883 }
 884
 885 static struct i915_vma *
 886 create_user_vma(struct i915_address_space *vm, unsigned long size)
 887 {
 888         struct drm_i915_gem_object *obj;
 889         struct i915_vma *vma;
 890         int err;
 891
 892         obj = i915_gem_object_create_internal(vm->i915, size);
 893         if (IS_ERR(obj))
 894                 return ERR_CAST(obj);
 895
 896         vma = i915_vma_instance(obj, vm, NULL);
 897         if (IS_ERR(vma)) {
 898                 i915_gem_object_put(obj);
 899                 return vma;
 900         }
 901
 902         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 903         if (err) {
 904                 i915_gem_object_put(obj);
 905                 return ERR_PTR(err);
 906         }
 907
 908         return vma;
 909 }
 910
 911 static struct i915_vma *
 912 store_context(struct intel_context *ce, struct i915_vma *scratch)
 913 {
 914         struct i915_vma *batch;
 915         u32 dw, x, *cs, *hw;
 916         u32 *defaults;
 917
 918         batch = create_user_vma(ce->vm, SZ_64K);
 919         if (IS_ERR(batch))
 920                 return batch;
 921
 922         cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 923         if (IS_ERR(cs)) {
 924                 i915_vma_put(batch);
 925                 return ERR_CAST(cs);
 926         }
 927
 928         defaults = shmem_pin_map(ce->engine->default_state);
 929         if (!defaults) {
 930                 i915_gem_object_unpin_map(batch->obj);
 931                 i915_vma_put(batch);
 932                 return ERR_PTR(-ENOMEM);
 933         }
 934
 935         x = 0;
 936         dw = 0;
 937         hw = defaults;
 938         hw += LRC_STATE_OFFSET / sizeof(*hw);
 939         do {
 940                 u32 len = hw[dw] & 0x7f;
 941
 942                 if (hw[dw] == 0) {
 943                         dw++;
 944                         continue;
 945                 }
 946
 947                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 948                         dw += len + 2;
 949                         continue;
 950                 }
 951
 952                 dw++;
 953                 len = (len + 1) / 2;
 954                 while (len--) {
 955                         *cs++ = MI_STORE_REGISTER_MEM_GEN8;
 956                         *cs++ = hw[dw];
 957                         *cs++ = lower_32_bits(scratch->node.start + x);
 958                         *cs++ = upper_32_bits(scratch->node.start + x);
 959
 960                         dw += 2;
 961                         x += 4;
 962                 }
 963         } while (dw < PAGE_SIZE / sizeof(u32) &&
 964                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 965
 966         *cs++ = MI_BATCH_BUFFER_END;
 967
 968         shmem_unpin_map(ce->engine->default_state, defaults);
 969
 970         i915_gem_object_flush_map(batch->obj);
 971         i915_gem_object_unpin_map(batch->obj);
 972
 973         return batch;
 974 }
 975
 976 static int move_to_active(struct i915_request *rq,
 977                           struct i915_vma *vma,
 978                           unsigned int flags)
 979 {
 980         int err;
 981
 982         i915_vma_lock(vma);
 983         err = i915_request_await_object(rq, vma->obj, flags);
 984         if (!err)
 985                 err = i915_vma_move_to_active(vma, rq, flags);
 986         i915_vma_unlock(vma);
 987
 988         return err;
 989 }
 990
 991 static struct i915_request *
 992 record_registers(struct intel_context *ce,
 993                  struct i915_vma *before,
 994                  struct i915_vma *after,
 995                  u32 *sema)
 996 {
 997         struct i915_vma *b_before, *b_after;
 998         struct i915_request *rq;
 999         u32 *cs;
1000         int err;
1001
1002         b_before = store_context(ce, before);
1003         if (IS_ERR(b_before))
1004                 return ERR_CAST(b_before);
1005
1006         b_after = store_context(ce, after);
1007         if (IS_ERR(b_after)) {
1008                 rq = ERR_CAST(b_after);
1009                 goto err_before;
1010         }
1011
1012         rq = intel_context_create_request(ce);
1013         if (IS_ERR(rq))
1014                 goto err_after;
1015
1016         err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1017         if (err)
1018                 goto err_rq;
1019
1020         err = move_to_active(rq, b_before, 0);
1021         if (err)
1022                 goto err_rq;
1023
1024         err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1025         if (err)
1026                 goto err_rq;
1027
1028         err = move_to_active(rq, b_after, 0);
1029         if (err)
1030                 goto err_rq;
1031
1032         cs = intel_ring_begin(rq, 14);
1033         if (IS_ERR(cs)) {
1034                 err = PTR_ERR(cs);
1035                 goto err_rq;
1036         }
1037
1038         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1039         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1040         *cs++ = lower_32_bits(b_before->node.start);
1041         *cs++ = upper_32_bits(b_before->node.start);
1042
1043         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1044         *cs++ = MI_SEMAPHORE_WAIT |
1045                 MI_SEMAPHORE_GLOBAL_GTT |
1046                 MI_SEMAPHORE_POLL |
1047                 MI_SEMAPHORE_SAD_NEQ_SDD;
1048         *cs++ = 0;
1049         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1050                 offset_in_page(sema);
1051         *cs++ = 0;
1052         *cs++ = MI_NOOP;
1053
1054         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1055         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1056         *cs++ = lower_32_bits(b_after->node.start);
1057         *cs++ = upper_32_bits(b_after->node.start);
1058
1059         intel_ring_advance(rq, cs);
1060
1061         WRITE_ONCE(*sema, 0);
1062         i915_request_get(rq);
1063         i915_request_add(rq);
1064 err_after:
1065         i915_vma_put(b_after);
1066 err_before:
1067         i915_vma_put(b_before);
1068         return rq;
1069
1070 err_rq:
1071         i915_request_add(rq);
1072         rq = ERR_PTR(err);
1073         goto err_after;
1074 }
1075
1076 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1077 {
1078         struct i915_vma *batch;
1079         u32 dw, *cs, *hw;
1080         u32 *defaults;
1081
1082         batch = create_user_vma(ce->vm, SZ_64K);
1083         if (IS_ERR(batch))
1084                 return batch;
1085
1086         cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
1087         if (IS_ERR(cs)) {
1088                 i915_vma_put(batch);
1089                 return ERR_CAST(cs);
1090         }
1091
1092         defaults = shmem_pin_map(ce->engine->default_state);
1093         if (!defaults) {
1094                 i915_gem_object_unpin_map(batch->obj);
1095                 i915_vma_put(batch);
1096                 return ERR_PTR(-ENOMEM);
1097         }
1098
1099         dw = 0;
1100         hw = defaults;
1101         hw += LRC_STATE_OFFSET / sizeof(*hw);
1102         do {
1103                 u32 len = hw[dw] & 0x7f;
1104
1105                 if (hw[dw] == 0) {
1106                         dw++;
1107                         continue;
1108                 }
1109
1110                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1111                         dw += len + 2;
1112                         continue;
1113                 }
1114
1115                 dw++;
1116                 len = (len + 1) / 2;
1117                 *cs++ = MI_LOAD_REGISTER_IMM(len);
1118                 while (len--) {
1119                         *cs++ = hw[dw];
1120                         *cs++ = poison;
1121                         dw += 2;
1122                 }
1123         } while (dw < PAGE_SIZE / sizeof(u32) &&
1124                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1125
1126         *cs++ = MI_BATCH_BUFFER_END;
1127
1128         shmem_unpin_map(ce->engine->default_state, defaults);
1129
1130         i915_gem_object_flush_map(batch->obj);
1131         i915_gem_object_unpin_map(batch->obj);
1132
1133         return batch;
1134 }
1135
1136 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1137 {
1138         struct i915_request *rq;
1139         struct i915_vma *batch;
1140         u32 *cs;
1141         int err;
1142
1143         batch = load_context(ce, poison);
1144         if (IS_ERR(batch))
1145                 return PTR_ERR(batch);
1146
1147         rq = intel_context_create_request(ce);
1148         if (IS_ERR(rq)) {
1149                 err = PTR_ERR(rq);
1150                 goto err_batch;
1151         }
1152
1153         err = move_to_active(rq, batch, 0);
1154         if (err)
1155                 goto err_rq;
1156
1157         cs = intel_ring_begin(rq, 8);
1158         if (IS_ERR(cs)) {
1159                 err = PTR_ERR(cs);
1160                 goto err_rq;
1161         }
1162
1163         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1164         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1165         *cs++ = lower_32_bits(batch->node.start);
1166         *cs++ = upper_32_bits(batch->node.start);
1167
1168         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1169         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1170                 offset_in_page(sema);
1171         *cs++ = 0;
1172         *cs++ = 1;
1173
1174         intel_ring_advance(rq, cs);
1175
1176         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1177 err_rq:
1178         i915_request_add(rq);
1179 err_batch:
1180         i915_vma_put(batch);
1181         return err;
1182 }
1183
1184 static bool is_moving(u32 a, u32 b)
1185 {
1186         return a != b;
1187 }
1188
1189 static int compare_isolation(struct intel_engine_cs *engine,
1190                              struct i915_vma *ref[2],
1191                              struct i915_vma *result[2],
1192                              struct intel_context *ce,
1193                              u32 poison)
1194 {
1195         u32 x, dw, *hw, *lrc;
1196         u32 *A[2], *B[2];
1197         u32 *defaults;
1198         int err = 0;
1199
1200         A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);
1201         if (IS_ERR(A[0]))
1202                 return PTR_ERR(A[0]);
1203
1204         A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);
1205         if (IS_ERR(A[1])) {
1206                 err = PTR_ERR(A[1]);
1207                 goto err_A0;
1208         }
1209
1210         B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);
1211         if (IS_ERR(B[0])) {
1212                 err = PTR_ERR(B[0]);
1213                 goto err_A1;
1214         }
1215
1216         B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);
1217         if (IS_ERR(B[1])) {
1218                 err = PTR_ERR(B[1]);
1219                 goto err_B0;
1220         }
1221
1222         lrc = i915_gem_object_pin_map(ce->state->obj,
1223                                       i915_coherent_map_type(engine->i915));
1224         if (IS_ERR(lrc)) {
1225                 err = PTR_ERR(lrc);
1226                 goto err_B1;
1227         }
1228         lrc += LRC_STATE_OFFSET / sizeof(*hw);
1229
1230         defaults = shmem_pin_map(ce->engine->default_state);
1231         if (!defaults) {
1232                 err = -ENOMEM;
1233                 goto err_lrc;
1234         }
1235
1236         x = 0;
1237         dw = 0;
1238         hw = defaults;
1239         hw += LRC_STATE_OFFSET / sizeof(*hw);
1240         do {
1241                 u32 len = hw[dw] & 0x7f;
1242
1243                 if (hw[dw] == 0) {
1244                         dw++;
1245                         continue;
1246                 }
1247
1248                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1249                         dw += len + 2;
1250                         continue;
1251                 }
1252
1253                 dw++;
1254                 len = (len + 1) / 2;
1255                 while (len--) {
1256                         if (!is_moving(A[0][x], A[1][x]) &&
1257                             (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1258                                 switch (hw[dw] & 4095) {
1259                                 case 0x30: /* RING_HEAD */
1260                                 case 0x34: /* RING_TAIL */
1261                                         break;
1262
1263                                 default:
1264                                         pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1265                                                engine->name, dw,
1266                                                hw[dw], hw[dw + 1],
1267                                                A[0][x], B[0][x], B[1][x],
1268                                                poison, lrc[dw + 1]);
1269                                         err = -EINVAL;
1270                                 }
1271                         }
1272                         dw += 2;
1273                         x++;
1274                 }
1275         } while (dw < PAGE_SIZE / sizeof(u32) &&
1276                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1277
1278         shmem_unpin_map(ce->engine->default_state, defaults);
1279 err_lrc:
1280         i915_gem_object_unpin_map(ce->state->obj);
1281 err_B1:
1282         i915_gem_object_unpin_map(result[1]->obj);
1283 err_B0:
1284         i915_gem_object_unpin_map(result[0]->obj);
1285 err_A1:
1286         i915_gem_object_unpin_map(ref[1]->obj);
1287 err_A0:
1288         i915_gem_object_unpin_map(ref[0]->obj);
1289         return err;
1290 }
1291
1292 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1293 {
1294         u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1295         struct i915_vma *ref[2], *result[2];
1296         struct intel_context *A, *B;
1297         struct i915_request *rq;
1298         int err;
1299
1300         A = intel_context_create(engine);
1301         if (IS_ERR(A))
1302                 return PTR_ERR(A);
1303
1304         B = intel_context_create(engine);
1305         if (IS_ERR(B)) {
1306                 err = PTR_ERR(B);
1307                 goto err_A;
1308         }
1309
1310         ref[0] = create_user_vma(A->vm, SZ_64K);
1311         if (IS_ERR(ref[0])) {
1312                 err = PTR_ERR(ref[0]);
1313                 goto err_B;
1314         }
1315
1316         ref[1] = create_user_vma(A->vm, SZ_64K);
1317         if (IS_ERR(ref[1])) {
1318                 err = PTR_ERR(ref[1]);
1319                 goto err_ref0;
1320         }
1321
1322         rq = record_registers(A, ref[0], ref[1], sema);
1323         if (IS_ERR(rq)) {
1324                 err = PTR_ERR(rq);
1325                 goto err_ref1;
1326         }
1327
1328         WRITE_ONCE(*sema, 1);
1329         wmb();
1330
1331         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1332                 i915_request_put(rq);
1333                 err = -ETIME;
1334                 goto err_ref1;
1335         }
1336         i915_request_put(rq);
1337
1338         result[0] = create_user_vma(A->vm, SZ_64K);
1339         if (IS_ERR(result[0])) {
1340                 err = PTR_ERR(result[0]);
1341                 goto err_ref1;
1342         }
1343
1344         result[1] = create_user_vma(A->vm, SZ_64K);
1345         if (IS_ERR(result[1])) {
1346                 err = PTR_ERR(result[1]);
1347                 goto err_result0;
1348         }
1349
1350         rq = record_registers(A, result[0], result[1], sema);
1351         if (IS_ERR(rq)) {
1352                 err = PTR_ERR(rq);
1353                 goto err_result1;
1354         }
1355
1356         err = poison_registers(B, poison, sema);
1357         if (err) {
1358                 WRITE_ONCE(*sema, -1);
1359                 i915_request_put(rq);
1360                 goto err_result1;
1361         }
1362
1363         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1364                 i915_request_put(rq);
1365                 err = -ETIME;
1366                 goto err_result1;
1367         }
1368         i915_request_put(rq);
1369
1370         err = compare_isolation(engine, ref, result, A, poison);
1371
1372 err_result1:
1373         i915_vma_put(result[1]);
1374 err_result0:
1375         i915_vma_put(result[0]);
1376 err_ref1:
1377         i915_vma_put(ref[1]);
1378 err_ref0:
1379         i915_vma_put(ref[0]);
1380 err_B:
1381         intel_context_put(B);
1382 err_A:
1383         intel_context_put(A);
1384         return err;
1385 }
1386
1387 static bool skip_isolation(const struct intel_engine_cs *engine)
1388 {
1389         if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
1390                 return true;
1391
1392         if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
1393                 return true;
1394
1395         return false;
1396 }
1397
1398 static int live_lrc_isolation(void *arg)
1399 {
1400         struct intel_gt *gt = arg;
1401         struct intel_engine_cs *engine;
1402         enum intel_engine_id id;
1403         const u32 poison[] = {
1404                 STACK_MAGIC,
1405                 0x3a3a3a3a,
1406                 0x5c5c5c5c,
1407                 0xffffffff,
1408                 0xffff0000,
1409         };
1410         int err = 0;
1411
1412         /*
1413          * Our goal is try and verify that per-context state cannot be
1414          * tampered with by another non-privileged client.
1415          *
1416          * We take the list of context registers from the LRI in the default
1417          * context image and attempt to modify that list from a remote context.
1418          */
1419
1420         for_each_engine(engine, gt, id) {
1421                 int i;
1422
1423                 /* Just don't even ask */
1424                 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1425                     skip_isolation(engine))
1426                         continue;
1427
1428                 intel_engine_pm_get(engine);
1429                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
1430                         int result;
1431
1432                         result = __lrc_isolation(engine, poison[i]);
1433                         if (result && !err)
1434                                 err = result;
1435
1436                         result = __lrc_isolation(engine, ~poison[i]);
1437                         if (result && !err)
1438                                 err = result;
1439                 }
1440                 intel_engine_pm_put(engine);
1441                 if (igt_flush_test(gt->i915)) {
1442                         err = -EIO;
1443                         break;
1444                 }
1445         }
1446
1447         return err;
1448 }
1449
1450 static int indirect_ctx_submit_req(struct intel_context *ce)
1451 {
1452         struct i915_request *rq;
1453         int err = 0;
1454
1455         rq = intel_context_create_request(ce);
1456         if (IS_ERR(rq))
1457                 return PTR_ERR(rq);
1458
1459         i915_request_get(rq);
1460         i915_request_add(rq);
1461
1462         if (i915_request_wait(rq, 0, HZ / 5) < 0)
1463                 err = -ETIME;
1464
1465         i915_request_put(rq);
1466
1467         return err;
1468 }
1469
1470 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1471 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1472
1473 static u32 *
1474 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1475 {
1476         *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1477                 MI_SRM_LRM_GLOBAL_GTT |
1478                 MI_LRI_LRM_CS_MMIO;
1479         *cs++ = i915_mmio_reg_offset(RING_START(0));
1480         *cs++ = i915_ggtt_offset(ce->state) +
1481                 context_wa_bb_offset(ce) +
1482                 CTX_BB_CANARY_OFFSET;
1483         *cs++ = 0;
1484
1485         return cs;
1486 }
1487
1488 static void
1489 indirect_ctx_bb_setup(struct intel_context *ce)
1490 {
1491         u32 *cs = context_indirect_bb(ce);
1492
1493         cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1494
1495         setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1496 }
1497
1498 static bool check_ring_start(struct intel_context *ce)
1499 {
1500         const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1501                 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1502
1503         if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1504                 return true;
1505
1506         pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1507                ctx_bb[CTX_BB_CANARY_INDEX],
1508                ce->lrc_reg_state[CTX_RING_START]);
1509
1510         return false;
1511 }
1512
1513 static int indirect_ctx_bb_check(struct intel_context *ce)
1514 {
1515         int err;
1516
1517         err = indirect_ctx_submit_req(ce);
1518         if (err)
1519                 return err;
1520
1521         if (!check_ring_start(ce))
1522                 return -EINVAL;
1523
1524         return 0;
1525 }
1526
1527 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1528 {
1529         struct intel_context *a, *b;
1530         int err;
1531
1532         a = intel_context_create(engine);
1533         if (IS_ERR(a))
1534                 return PTR_ERR(a);
1535         err = intel_context_pin(a);
1536         if (err)
1537                 goto put_a;
1538
1539         b = intel_context_create(engine);
1540         if (IS_ERR(b)) {
1541                 err = PTR_ERR(b);
1542                 goto unpin_a;
1543         }
1544         err = intel_context_pin(b);
1545         if (err)
1546                 goto put_b;
1547
1548         /* We use the already reserved extra page in context state */
1549         if (!a->wa_bb_page) {
1550                 GEM_BUG_ON(b->wa_bb_page);
1551                 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12);
1552                 goto unpin_b;
1553         }
1554
1555         /*
1556          * In order to test that our per context bb is truly per context,
1557          * and executes at the intended spot on context restoring process,
1558          * make the batch store the ring start value to memory.
1559          * As ring start is restored apriori of starting the indirect ctx bb and
1560          * as it will be different for each context, it fits to this purpose.
1561          */
1562         indirect_ctx_bb_setup(a);
1563         indirect_ctx_bb_setup(b);
1564
1565         err = indirect_ctx_bb_check(a);
1566         if (err)
1567                 goto unpin_b;
1568
1569         err = indirect_ctx_bb_check(b);
1570
1571 unpin_b:
1572         intel_context_unpin(b);
1573 put_b:
1574         intel_context_put(b);
1575 unpin_a:
1576         intel_context_unpin(a);
1577 put_a:
1578         intel_context_put(a);
1579
1580         return err;
1581 }
1582
1583 static int live_lrc_indirect_ctx_bb(void *arg)
1584 {
1585         struct intel_gt *gt = arg;
1586         struct intel_engine_cs *engine;
1587         enum intel_engine_id id;
1588         int err = 0;
1589
1590         for_each_engine(engine, gt, id) {
1591                 intel_engine_pm_get(engine);
1592                 err = __live_lrc_indirect_ctx_bb(engine);
1593                 intel_engine_pm_put(engine);
1594
1595                 if (igt_flush_test(gt->i915))
1596                         err = -EIO;
1597
1598                 if (err)
1599                         break;
1600         }
1601
1602         return err;
1603 }
1604
1605 static void garbage_reset(struct intel_engine_cs *engine,
1606                           struct i915_request *rq)
1607 {
1608         const unsigned int bit = I915_RESET_ENGINE + engine->id;
1609         unsigned long *lock = &engine->gt->reset.flags;
1610
1611         local_bh_disable();
1612         if (!test_and_set_bit(bit, lock)) {
1613                 tasklet_disable(&engine->execlists.tasklet);
1614
1615                 if (!rq->fence.error)
1616                         __intel_engine_reset_bh(engine, NULL);
1617
1618                 tasklet_enable(&engine->execlists.tasklet);
1619                 clear_and_wake_up_bit(bit, lock);
1620         }
1621         local_bh_enable();
1622 }
1623
1624 static struct i915_request *garbage(struct intel_context *ce,
1625                                     struct rnd_state *prng)
1626 {
1627         struct i915_request *rq;
1628         int err;
1629
1630         err = intel_context_pin(ce);
1631         if (err)
1632                 return ERR_PTR(err);
1633
1634         prandom_bytes_state(prng,
1635                             ce->lrc_reg_state,
1636                             ce->engine->context_size -
1637                             LRC_STATE_OFFSET);
1638
1639         rq = intel_context_create_request(ce);
1640         if (IS_ERR(rq)) {
1641                 err = PTR_ERR(rq);
1642                 goto err_unpin;
1643         }
1644
1645         i915_request_get(rq);
1646         i915_request_add(rq);
1647         return rq;
1648
1649 err_unpin:
1650         intel_context_unpin(ce);
1651         return ERR_PTR(err);
1652 }
1653
1654 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1655 {
1656         struct intel_context *ce;
1657         struct i915_request *hang;
1658         int err = 0;
1659
1660         ce = intel_context_create(engine);
1661         if (IS_ERR(ce))
1662                 return PTR_ERR(ce);
1663
1664         hang = garbage(ce, prng);
1665         if (IS_ERR(hang)) {
1666                 err = PTR_ERR(hang);
1667                 goto err_ce;
1668         }
1669
1670         if (wait_for_submit(engine, hang, HZ / 2)) {
1671                 i915_request_put(hang);
1672                 err = -ETIME;
1673                 goto err_ce;
1674         }
1675
1676         intel_context_set_banned(ce);
1677         garbage_reset(engine, hang);
1678
1679         intel_engine_flush_submission(engine);
1680         if (!hang->fence.error) {
1681                 i915_request_put(hang);
1682                 pr_err("%s: corrupted context was not reset\n",
1683                        engine->name);
1684                 err = -EINVAL;
1685                 goto err_ce;
1686         }
1687
1688         if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1689                 pr_err("%s: corrupted context did not recover\n",
1690                        engine->name);
1691                 i915_request_put(hang);
1692                 err = -EIO;
1693                 goto err_ce;
1694         }
1695         i915_request_put(hang);
1696
1697 err_ce:
1698         intel_context_put(ce);
1699         return err;
1700 }
1701
1702 static int live_lrc_garbage(void *arg)
1703 {
1704         struct intel_gt *gt = arg;
1705         struct intel_engine_cs *engine;
1706         enum intel_engine_id id;
1707
1708         /*
1709          * Verify that we can recover if one context state is completely
1710          * corrupted.
1711          */
1712
1713         if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1714                 return 0;
1715
1716         for_each_engine(engine, gt, id) {
1717                 I915_RND_STATE(prng);
1718                 int err = 0, i;
1719
1720                 if (!intel_has_reset_engine(engine->gt))
1721                         continue;
1722
1723                 intel_engine_pm_get(engine);
1724                 for (i = 0; i < 3; i++) {
1725                         err = __lrc_garbage(engine, &prng);
1726                         if (err)
1727                                 break;
1728                 }
1729                 intel_engine_pm_put(engine);
1730
1731                 if (igt_flush_test(gt->i915))
1732                         err = -EIO;
1733                 if (err)
1734                         return err;
1735         }
1736
1737         return 0;
1738 }
1739
1740 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1741 {
1742         struct intel_context *ce;
1743         struct i915_request *rq;
1744         IGT_TIMEOUT(end_time);
1745         int err;
1746
1747         ce = intel_context_create(engine);
1748         if (IS_ERR(ce))
1749                 return PTR_ERR(ce);
1750
1751         ce->runtime.num_underflow = 0;
1752         ce->runtime.max_underflow = 0;
1753
1754         do {
1755                 unsigned int loop = 1024;
1756
1757                 while (loop) {
1758                         rq = intel_context_create_request(ce);
1759                         if (IS_ERR(rq)) {
1760                                 err = PTR_ERR(rq);
1761                                 goto err_rq;
1762                         }
1763
1764                         if (--loop == 0)
1765                                 i915_request_get(rq);
1766
1767                         i915_request_add(rq);
1768                 }
1769
1770                 if (__igt_timeout(end_time, NULL))
1771                         break;
1772
1773                 i915_request_put(rq);
1774         } while (1);
1775
1776         err = i915_request_wait(rq, 0, HZ / 5);
1777         if (err < 0) {
1778                 pr_err("%s: request not completed!\n", engine->name);
1779                 goto err_wait;
1780         }
1781
1782         igt_flush_test(engine->i915);
1783
1784         pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1785                 engine->name,
1786                 intel_context_get_total_runtime_ns(ce),
1787                 intel_context_get_avg_runtime_ns(ce));
1788
1789         err = 0;
1790         if (ce->runtime.num_underflow) {
1791                 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1792                        engine->name,
1793                        ce->runtime.num_underflow,
1794                        ce->runtime.max_underflow);
1795                 GEM_TRACE_DUMP();
1796                 err = -EOVERFLOW;
1797         }
1798
1799 err_wait:
1800         i915_request_put(rq);
1801 err_rq:
1802         intel_context_put(ce);
1803         return err;
1804 }
1805
1806 static int live_pphwsp_runtime(void *arg)
1807 {
1808         struct intel_gt *gt = arg;
1809         struct intel_engine_cs *engine;
1810         enum intel_engine_id id;
1811         int err = 0;
1812
1813         /*
1814          * Check that cumulative context runtime as stored in the pphwsp[16]
1815          * is monotonic.
1816          */
1817
1818         for_each_engine(engine, gt, id) {
1819                 err = __live_pphwsp_runtime(engine);
1820                 if (err)
1821                         break;
1822         }
1823
1824         if (igt_flush_test(gt->i915))
1825                 err = -EIO;
1826
1827         return err;
1828 }
1829
1830 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1831 {
1832         static const struct i915_subtest tests[] = {
1833                 SUBTEST(live_lrc_layout),
1834                 SUBTEST(live_lrc_fixed),
1835                 SUBTEST(live_lrc_state),
1836                 SUBTEST(live_lrc_gpr),
1837                 SUBTEST(live_lrc_isolation),
1838                 SUBTEST(live_lrc_timestamp),
1839                 SUBTEST(live_lrc_garbage),
1840                 SUBTEST(live_pphwsp_runtime),
1841                 SUBTEST(live_lrc_indirect_ctx_bb),
1842         };
1843
1844         if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1845                 return 0;
1846
1847         return intel_gt_live_subtests(tests, &i915->gt);
1848 }