drivers/gpu/drm/i915/gt/selftest_lrc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include <linux/prime_numbers.h>
   7
   8 #include "i915_selftest.h"
   9 #include "intel_engine_heartbeat.h"
  10 #include "intel_engine_pm.h"
  11 #include "intel_reset.h"
  12 #include "intel_ring.h"
  13 #include "selftest_engine_heartbeat.h"
  14 #include "selftests/i915_random.h"
  15 #include "selftests/igt_flush_test.h"
  16 #include "selftests/igt_live_test.h"
  17 #include "selftests/igt_spinner.h"
  18 #include "selftests/lib_sw_fence.h"
  19 #include "shmem_utils.h"
  20
  21 #include "gem/selftests/igt_gem_utils.h"
  22 #include "gem/selftests/mock_context.h"
  23
  24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
  25 #define NUM_GPR 16
  26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
  27
  28 static struct i915_vma *create_scratch(struct intel_gt *gt)
  29 {
  30         return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
  31 }
  32
  33 static bool is_active(struct i915_request *rq)
  34 {
  35         if (i915_request_is_active(rq))
  36                 return true;
  37
  38         if (i915_request_on_hold(rq))
  39                 return true;
  40
  41         if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
  42                 return true;
  43
  44         return false;
  45 }
  46
  47 static int wait_for_submit(struct intel_engine_cs *engine,
  48                            struct i915_request *rq,
  49                            unsigned long timeout)
  50 {
  51         /* Ignore our own attempts to suppress excess tasklets */
  52         tasklet_hi_schedule(&engine->sched_engine->tasklet);
  53
  54         timeout += jiffies;
  55         do {
  56                 bool done = time_after(jiffies, timeout);
  57
  58                 if (i915_request_completed(rq)) /* that was quick! */
  59                         return 0;
  60
  61                 /* Wait until the HW has acknowleged the submission (or err) */
  62                 intel_engine_flush_submission(engine);
  63                 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
  64                         return 0;
  65
  66                 if (done)
  67                         return -ETIME;
  68
  69                 cond_resched();
  70         } while (1);
  71 }
  72
  73 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
  74 {
  75         const u32 offset =
  76                 i915_ggtt_offset(ce->engine->status_page.vma) +
  77                 offset_in_page(slot);
  78         struct i915_request *rq;
  79         u32 *cs;
  80
  81         rq = intel_context_create_request(ce);
  82         if (IS_ERR(rq))
  83                 return PTR_ERR(rq);
  84
  85         cs = intel_ring_begin(rq, 4);
  86         if (IS_ERR(cs)) {
  87                 i915_request_add(rq);
  88                 return PTR_ERR(cs);
  89         }
  90
  91         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  92         *cs++ = offset;
  93         *cs++ = 0;
  94         *cs++ = 1;
  95
  96         intel_ring_advance(rq, cs);
  97
  98         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
  99         i915_request_add(rq);
 100         return 0;
 101 }
 102
 103 static int context_flush(struct intel_context *ce, long timeout)
 104 {
 105         struct i915_request *rq;
 106         struct dma_fence *fence;
 107         int err = 0;
 108
 109         rq = intel_engine_create_kernel_request(ce->engine);
 110         if (IS_ERR(rq))
 111                 return PTR_ERR(rq);
 112
 113         fence = i915_active_fence_get(&ce->timeline->last_request);
 114         if (fence) {
 115                 i915_request_await_dma_fence(rq, fence);
 116                 dma_fence_put(fence);
 117         }
 118
 119         rq = i915_request_get(rq);
 120         i915_request_add(rq);
 121         if (i915_request_wait(rq, 0, timeout) < 0)
 122                 err = -ETIME;
 123         i915_request_put(rq);
 124
 125         rmb(); /* We know the request is written, make sure all state is too! */
 126         return err;
 127 }
 128
 129 static int live_lrc_layout(void *arg)
 130 {
 131         struct intel_gt *gt = arg;
 132         struct intel_engine_cs *engine;
 133         enum intel_engine_id id;
 134         u32 *lrc;
 135         int err;
 136
 137         /*
 138          * Check the registers offsets we use to create the initial reg state
 139          * match the layout saved by HW.
 140          */
 141
 142         lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
 143         if (!lrc)
 144                 return -ENOMEM;
 145         GEM_BUG_ON(offset_in_page(lrc));
 146
 147         err = 0;
 148         for_each_engine(engine, gt, id) {
 149                 u32 *hw;
 150                 int dw;
 151
 152                 if (!engine->default_state)
 153                         continue;
 154
 155                 hw = shmem_pin_map(engine->default_state);
 156                 if (IS_ERR(hw)) {
 157                         err = PTR_ERR(hw);
 158                         break;
 159                 }
 160                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 161
 162                 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
 163                                 engine->kernel_context, engine, true);
 164
 165                 dw = 0;
 166                 do {
 167                         u32 lri = READ_ONCE(hw[dw]);
 168
 169                         if (lri == 0) {
 170                                 dw++;
 171                                 continue;
 172                         }
 173
 174                         if (lrc[dw] == 0) {
 175                                 pr_debug("%s: skipped instruction %x at dword %d\n",
 176                                          engine->name, lri, dw);
 177                                 dw++;
 178                                 continue;
 179                         }
 180
 181                         if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 182                                 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
 183                                        engine->name, dw, lri);
 184                                 err = -EINVAL;
 185                                 break;
 186                         }
 187
 188                         if (lrc[dw] != lri) {
 189                                 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
 190                                        engine->name, dw, lri, lrc[dw]);
 191                                 err = -EINVAL;
 192                                 break;
 193                         }
 194
 195                         lri &= 0x7f;
 196                         lri++;
 197                         dw++;
 198
 199                         while (lri) {
 200                                 u32 offset = READ_ONCE(hw[dw]);
 201
 202                                 if (offset != lrc[dw]) {
 203                                         pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
 204                                                engine->name, dw, offset, lrc[dw]);
 205                                         err = -EINVAL;
 206                                         break;
 207                                 }
 208
 209                                 /*
 210                                  * Skip over the actual register value as we
 211                                  * expect that to differ.
 212                                  */
 213                                 dw += 2;
 214                                 lri -= 2;
 215                         }
 216                 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 217
 218                 if (err) {
 219                         pr_info("%s: HW register image:\n", engine->name);
 220                         igt_hexdump(hw, PAGE_SIZE);
 221
 222                         pr_info("%s: SW register image:\n", engine->name);
 223                         igt_hexdump(lrc, PAGE_SIZE);
 224                 }
 225
 226                 shmem_unpin_map(engine->default_state, hw);
 227                 if (err)
 228                         break;
 229         }
 230
 231         free_page((unsigned long)lrc);
 232         return err;
 233 }
 234
 235 static int find_offset(const u32 *lri, u32 offset)
 236 {
 237         int i;
 238
 239         for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
 240                 if (lri[i] == offset)
 241                         return i;
 242
 243         return -1;
 244 }
 245
 246 static int live_lrc_fixed(void *arg)
 247 {
 248         struct intel_gt *gt = arg;
 249         struct intel_engine_cs *engine;
 250         enum intel_engine_id id;
 251         int err = 0;
 252
 253         /*
 254          * Check the assumed register offsets match the actual locations in
 255          * the context image.
 256          */
 257
 258         for_each_engine(engine, gt, id) {
 259                 const struct {
 260                         u32 reg;
 261                         u32 offset;
 262                         const char *name;
 263                 } tbl[] = {
 264                         {
 265                                 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
 266                                 CTX_RING_START - 1,
 267                                 "RING_START"
 268                         },
 269                         {
 270                                 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
 271                                 CTX_RING_CTL - 1,
 272                                 "RING_CTL"
 273                         },
 274                         {
 275                                 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
 276                                 CTX_RING_HEAD - 1,
 277                                 "RING_HEAD"
 278                         },
 279                         {
 280                                 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
 281                                 CTX_RING_TAIL - 1,
 282                                 "RING_TAIL"
 283                         },
 284                         {
 285                                 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
 286                                 lrc_ring_mi_mode(engine),
 287                                 "RING_MI_MODE"
 288                         },
 289                         {
 290                                 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
 291                                 CTX_BB_STATE - 1,
 292                                 "BB_STATE"
 293                         },
 294                         {
 295                                 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
 296                                 lrc_ring_wa_bb_per_ctx(engine),
 297                                 "RING_BB_PER_CTX_PTR"
 298                         },
 299                         {
 300                                 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
 301                                 lrc_ring_indirect_ptr(engine),
 302                                 "RING_INDIRECT_CTX_PTR"
 303                         },
 304                         {
 305                                 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
 306                                 lrc_ring_indirect_offset(engine),
 307                                 "RING_INDIRECT_CTX_OFFSET"
 308                         },
 309                         {
 310                                 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
 311                                 CTX_TIMESTAMP - 1,
 312                                 "RING_CTX_TIMESTAMP"
 313                         },
 314                         {
 315                                 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
 316                                 lrc_ring_gpr0(engine),
 317                                 "RING_CS_GPR0"
 318                         },
 319                         {
 320                                 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
 321                                 lrc_ring_cmd_buf_cctl(engine),
 322                                 "RING_CMD_BUF_CCTL"
 323                         },
 324                         { },
 325                 }, *t;
 326                 u32 *hw;
 327
 328                 if (!engine->default_state)
 329                         continue;
 330
 331                 hw = shmem_pin_map(engine->default_state);
 332                 if (IS_ERR(hw)) {
 333                         err = PTR_ERR(hw);
 334                         break;
 335                 }
 336                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 337
 338                 for (t = tbl; t->name; t++) {
 339                         int dw = find_offset(hw, t->reg);
 340
 341                         if (dw != t->offset) {
 342                                 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
 343                                        engine->name,
 344                                        t->name,
 345                                        t->reg,
 346                                        dw,
 347                                        t->offset);
 348                                 err = -EINVAL;
 349                         }
 350                 }
 351
 352                 shmem_unpin_map(engine->default_state, hw);
 353         }
 354
 355         return err;
 356 }
 357
 358 static int __live_lrc_state(struct intel_engine_cs *engine,
 359                             struct i915_vma *scratch)
 360 {
 361         struct intel_context *ce;
 362         struct i915_request *rq;
 363         struct i915_gem_ww_ctx ww;
 364         enum {
 365                 RING_START_IDX = 0,
 366                 RING_TAIL_IDX,
 367                 MAX_IDX
 368         };
 369         u32 expected[MAX_IDX];
 370         u32 *cs;
 371         int err;
 372         int n;
 373
 374         ce = intel_context_create(engine);
 375         if (IS_ERR(ce))
 376                 return PTR_ERR(ce);
 377
 378         i915_gem_ww_ctx_init(&ww, false);
 379 retry:
 380         err = i915_gem_object_lock(scratch->obj, &ww);
 381         if (!err)
 382                 err = intel_context_pin_ww(ce, &ww);
 383         if (err)
 384                 goto err_put;
 385
 386         rq = i915_request_create(ce);
 387         if (IS_ERR(rq)) {
 388                 err = PTR_ERR(rq);
 389                 goto err_unpin;
 390         }
 391
 392         cs = intel_ring_begin(rq, 4 * MAX_IDX);
 393         if (IS_ERR(cs)) {
 394                 err = PTR_ERR(cs);
 395                 i915_request_add(rq);
 396                 goto err_unpin;
 397         }
 398
 399         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 400         *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
 401         *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
 402         *cs++ = 0;
 403
 404         expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
 405
 406         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 407         *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
 408         *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
 409         *cs++ = 0;
 410
 411         err = i915_request_await_object(rq, scratch->obj, true);
 412         if (!err)
 413                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 414
 415         i915_request_get(rq);
 416         i915_request_add(rq);
 417         if (err)
 418                 goto err_rq;
 419
 420         intel_engine_flush_submission(engine);
 421         expected[RING_TAIL_IDX] = ce->ring->tail;
 422
 423         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 424                 err = -ETIME;
 425                 goto err_rq;
 426         }
 427
 428         cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 429         if (IS_ERR(cs)) {
 430                 err = PTR_ERR(cs);
 431                 goto err_rq;
 432         }
 433
 434         for (n = 0; n < MAX_IDX; n++) {
 435                 if (cs[n] != expected[n]) {
 436                         pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
 437                                engine->name, n, cs[n], expected[n]);
 438                         err = -EINVAL;
 439                         break;
 440                 }
 441         }
 442
 443         i915_gem_object_unpin_map(scratch->obj);
 444
 445 err_rq:
 446         i915_request_put(rq);
 447 err_unpin:
 448         intel_context_unpin(ce);
 449 err_put:
 450         if (err == -EDEADLK) {
 451                 err = i915_gem_ww_ctx_backoff(&ww);
 452                 if (!err)
 453                         goto retry;
 454         }
 455         i915_gem_ww_ctx_fini(&ww);
 456         intel_context_put(ce);
 457         return err;
 458 }
 459
 460 static int live_lrc_state(void *arg)
 461 {
 462         struct intel_gt *gt = arg;
 463         struct intel_engine_cs *engine;
 464         struct i915_vma *scratch;
 465         enum intel_engine_id id;
 466         int err = 0;
 467
 468         /*
 469          * Check the live register state matches what we expect for this
 470          * intel_context.
 471          */
 472
 473         scratch = create_scratch(gt);
 474         if (IS_ERR(scratch))
 475                 return PTR_ERR(scratch);
 476
 477         for_each_engine(engine, gt, id) {
 478                 err = __live_lrc_state(engine, scratch);
 479                 if (err)
 480                         break;
 481         }
 482
 483         if (igt_flush_test(gt->i915))
 484                 err = -EIO;
 485
 486         i915_vma_unpin_and_release(&scratch, 0);
 487         return err;
 488 }
 489
 490 static int gpr_make_dirty(struct intel_context *ce)
 491 {
 492         struct i915_request *rq;
 493         u32 *cs;
 494         int n;
 495
 496         rq = intel_context_create_request(ce);
 497         if (IS_ERR(rq))
 498                 return PTR_ERR(rq);
 499
 500         cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
 501         if (IS_ERR(cs)) {
 502                 i915_request_add(rq);
 503                 return PTR_ERR(cs);
 504         }
 505
 506         *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
 507         for (n = 0; n < NUM_GPR_DW; n++) {
 508                 *cs++ = CS_GPR(ce->engine, n);
 509                 *cs++ = STACK_MAGIC;
 510         }
 511         *cs++ = MI_NOOP;
 512
 513         intel_ring_advance(rq, cs);
 514
 515         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
 516         i915_request_add(rq);
 517
 518         return 0;
 519 }
 520
 521 static struct i915_request *
 522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
 523 {
 524         const u32 offset =
 525                 i915_ggtt_offset(ce->engine->status_page.vma) +
 526                 offset_in_page(slot);
 527         struct i915_request *rq;
 528         u32 *cs;
 529         int err;
 530         int n;
 531
 532         rq = intel_context_create_request(ce);
 533         if (IS_ERR(rq))
 534                 return rq;
 535
 536         cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
 537         if (IS_ERR(cs)) {
 538                 i915_request_add(rq);
 539                 return ERR_CAST(cs);
 540         }
 541
 542         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 543         *cs++ = MI_NOOP;
 544
 545         *cs++ = MI_SEMAPHORE_WAIT |
 546                 MI_SEMAPHORE_GLOBAL_GTT |
 547                 MI_SEMAPHORE_POLL |
 548                 MI_SEMAPHORE_SAD_NEQ_SDD;
 549         *cs++ = 0;
 550         *cs++ = offset;
 551         *cs++ = 0;
 552
 553         for (n = 0; n < NUM_GPR_DW; n++) {
 554                 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 555                 *cs++ = CS_GPR(ce->engine, n);
 556                 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
 557                 *cs++ = 0;
 558         }
 559
 560         i915_vma_lock(scratch);
 561         err = i915_request_await_object(rq, scratch->obj, true);
 562         if (!err)
 563                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 564         i915_vma_unlock(scratch);
 565
 566         i915_request_get(rq);
 567         i915_request_add(rq);
 568         if (err) {
 569                 i915_request_put(rq);
 570                 rq = ERR_PTR(err);
 571         }
 572
 573         return rq;
 574 }
 575
 576 static int __live_lrc_gpr(struct intel_engine_cs *engine,
 577                           struct i915_vma *scratch,
 578                           bool preempt)
 579 {
 580         u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
 581         struct intel_context *ce;
 582         struct i915_request *rq;
 583         u32 *cs;
 584         int err;
 585         int n;
 586
 587         if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
 588                 return 0; /* GPR only on rcs0 for gen8 */
 589
 590         err = gpr_make_dirty(engine->kernel_context);
 591         if (err)
 592                 return err;
 593
 594         ce = intel_context_create(engine);
 595         if (IS_ERR(ce))
 596                 return PTR_ERR(ce);
 597
 598         rq = __gpr_read(ce, scratch, slot);
 599         if (IS_ERR(rq)) {
 600                 err = PTR_ERR(rq);
 601                 goto err_put;
 602         }
 603
 604         err = wait_for_submit(engine, rq, HZ / 2);
 605         if (err)
 606                 goto err_rq;
 607
 608         if (preempt) {
 609                 err = gpr_make_dirty(engine->kernel_context);
 610                 if (err)
 611                         goto err_rq;
 612
 613                 err = emit_semaphore_signal(engine->kernel_context, slot);
 614                 if (err)
 615                         goto err_rq;
 616
 617                 err = wait_for_submit(engine, rq, HZ / 2);
 618                 if (err)
 619                         goto err_rq;
 620         } else {
 621                 slot[0] = 1;
 622                 wmb();
 623         }
 624
 625         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 626                 err = -ETIME;
 627                 goto err_rq;
 628         }
 629
 630         cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
 631         if (IS_ERR(cs)) {
 632                 err = PTR_ERR(cs);
 633                 goto err_rq;
 634         }
 635
 636         for (n = 0; n < NUM_GPR_DW; n++) {
 637                 if (cs[n]) {
 638                         pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
 639                                engine->name,
 640                                n / 2, n & 1 ? "udw" : "ldw",
 641                                cs[n]);
 642                         err = -EINVAL;
 643                         break;
 644                 }
 645         }
 646
 647         i915_gem_object_unpin_map(scratch->obj);
 648
 649 err_rq:
 650         memset32(&slot[0], -1, 4);
 651         wmb();
 652         i915_request_put(rq);
 653 err_put:
 654         intel_context_put(ce);
 655         return err;
 656 }
 657
 658 static int live_lrc_gpr(void *arg)
 659 {
 660         struct intel_gt *gt = arg;
 661         struct intel_engine_cs *engine;
 662         struct i915_vma *scratch;
 663         enum intel_engine_id id;
 664         int err = 0;
 665
 666         /*
 667          * Check that GPR registers are cleared in new contexts as we need
 668          * to avoid leaking any information from previous contexts.
 669          */
 670
 671         scratch = create_scratch(gt);
 672         if (IS_ERR(scratch))
 673                 return PTR_ERR(scratch);
 674
 675         for_each_engine(engine, gt, id) {
 676                 st_engine_heartbeat_disable(engine);
 677
 678                 err = __live_lrc_gpr(engine, scratch, false);
 679                 if (err)
 680                         goto err;
 681
 682                 err = __live_lrc_gpr(engine, scratch, true);
 683                 if (err)
 684                         goto err;
 685
 686 err:
 687                 st_engine_heartbeat_enable(engine);
 688                 if (igt_flush_test(gt->i915))
 689                         err = -EIO;
 690                 if (err)
 691                         break;
 692         }
 693
 694         i915_vma_unpin_and_release(&scratch, 0);
 695         return err;
 696 }
 697
 698 static struct i915_request *
 699 create_timestamp(struct intel_context *ce, void *slot, int idx)
 700 {
 701         const u32 offset =
 702                 i915_ggtt_offset(ce->engine->status_page.vma) +
 703                 offset_in_page(slot);
 704         struct i915_request *rq;
 705         u32 *cs;
 706         int err;
 707
 708         rq = intel_context_create_request(ce);
 709         if (IS_ERR(rq))
 710                 return rq;
 711
 712         cs = intel_ring_begin(rq, 10);
 713         if (IS_ERR(cs)) {
 714                 err = PTR_ERR(cs);
 715                 goto err;
 716         }
 717
 718         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 719         *cs++ = MI_NOOP;
 720
 721         *cs++ = MI_SEMAPHORE_WAIT |
 722                 MI_SEMAPHORE_GLOBAL_GTT |
 723                 MI_SEMAPHORE_POLL |
 724                 MI_SEMAPHORE_SAD_NEQ_SDD;
 725         *cs++ = 0;
 726         *cs++ = offset;
 727         *cs++ = 0;
 728
 729         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 730         *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
 731         *cs++ = offset + idx * sizeof(u32);
 732         *cs++ = 0;
 733
 734         intel_ring_advance(rq, cs);
 735
 736         err = 0;
 737 err:
 738         i915_request_get(rq);
 739         i915_request_add(rq);
 740         if (err) {
 741                 i915_request_put(rq);
 742                 return ERR_PTR(err);
 743         }
 744
 745         return rq;
 746 }
 747
 748 struct lrc_timestamp {
 749         struct intel_engine_cs *engine;
 750         struct intel_context *ce[2];
 751         u32 poison;
 752 };
 753
 754 static bool timestamp_advanced(u32 start, u32 end)
 755 {
 756         return (s32)(end - start) > 0;
 757 }
 758
 759 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
 760 {
 761         u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
 762         struct i915_request *rq;
 763         u32 timestamp;
 764         int err = 0;
 765
 766         arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
 767         rq = create_timestamp(arg->ce[0], slot, 1);
 768         if (IS_ERR(rq))
 769                 return PTR_ERR(rq);
 770
 771         err = wait_for_submit(rq->engine, rq, HZ / 2);
 772         if (err)
 773                 goto err;
 774
 775         if (preempt) {
 776                 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
 777                 err = emit_semaphore_signal(arg->ce[1], slot);
 778                 if (err)
 779                         goto err;
 780         } else {
 781                 slot[0] = 1;
 782                 wmb();
 783         }
 784
 785         /* And wait for switch to kernel (to save our context to memory) */
 786         err = context_flush(arg->ce[0], HZ / 2);
 787         if (err)
 788                 goto err;
 789
 790         if (!timestamp_advanced(arg->poison, slot[1])) {
 791                 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
 792                        arg->engine->name, preempt ? "preempt" : "simple",
 793                        arg->poison, slot[1]);
 794                 err = -EINVAL;
 795         }
 796
 797         timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
 798         if (!timestamp_advanced(slot[1], timestamp)) {
 799                 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
 800                        arg->engine->name, preempt ? "preempt" : "simple",
 801                        slot[1], timestamp);
 802                 err = -EINVAL;
 803         }
 804
 805 err:
 806         memset32(slot, -1, 4);
 807         i915_request_put(rq);
 808         return err;
 809 }
 810
 811 static int live_lrc_timestamp(void *arg)
 812 {
 813         struct lrc_timestamp data = {};
 814         struct intel_gt *gt = arg;
 815         enum intel_engine_id id;
 816         const u32 poison[] = {
 817                 0,
 818                 S32_MAX,
 819                 (u32)S32_MAX + 1,
 820                 U32_MAX,
 821         };
 822
 823         /*
 824          * We want to verify that the timestamp is saved and restore across
 825          * context switches and is monotonic.
 826          *
 827          * So we do this with a little bit of LRC poisoning to check various
 828          * boundary conditions, and see what happens if we preempt the context
 829          * with a second request (carrying more poison into the timestamp).
 830          */
 831
 832         for_each_engine(data.engine, gt, id) {
 833                 int i, err = 0;
 834
 835                 st_engine_heartbeat_disable(data.engine);
 836
 837                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 838                         struct intel_context *tmp;
 839
 840                         tmp = intel_context_create(data.engine);
 841                         if (IS_ERR(tmp)) {
 842                                 err = PTR_ERR(tmp);
 843                                 goto err;
 844                         }
 845
 846                         err = intel_context_pin(tmp);
 847                         if (err) {
 848                                 intel_context_put(tmp);
 849                                 goto err;
 850                         }
 851
 852                         data.ce[i] = tmp;
 853                 }
 854
 855                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
 856                         data.poison = poison[i];
 857
 858                         err = __lrc_timestamp(&data, false);
 859                         if (err)
 860                                 break;
 861
 862                         err = __lrc_timestamp(&data, true);
 863                         if (err)
 864                                 break;
 865                 }
 866
 867 err:
 868                 st_engine_heartbeat_enable(data.engine);
 869                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 870                         if (!data.ce[i])
 871                                 break;
 872
 873                         intel_context_unpin(data.ce[i]);
 874                         intel_context_put(data.ce[i]);
 875                 }
 876
 877                 if (igt_flush_test(gt->i915))
 878                         err = -EIO;
 879                 if (err)
 880                         return err;
 881         }
 882
 883         return 0;
 884 }
 885
 886 static struct i915_vma *
 887 create_user_vma(struct i915_address_space *vm, unsigned long size)
 888 {
 889         struct drm_i915_gem_object *obj;
 890         struct i915_vma *vma;
 891         int err;
 892
 893         obj = i915_gem_object_create_internal(vm->i915, size);
 894         if (IS_ERR(obj))
 895                 return ERR_CAST(obj);
 896
 897         vma = i915_vma_instance(obj, vm, NULL);
 898         if (IS_ERR(vma)) {
 899                 i915_gem_object_put(obj);
 900                 return vma;
 901         }
 902
 903         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 904         if (err) {
 905                 i915_gem_object_put(obj);
 906                 return ERR_PTR(err);
 907         }
 908
 909         return vma;
 910 }
 911
 912 static struct i915_vma *
 913 store_context(struct intel_context *ce, struct i915_vma *scratch)
 914 {
 915         struct i915_vma *batch;
 916         u32 dw, x, *cs, *hw;
 917         u32 *defaults;
 918
 919         batch = create_user_vma(ce->vm, SZ_64K);
 920         if (IS_ERR(batch))
 921                 return batch;
 922
 923         cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
 924         if (IS_ERR(cs)) {
 925                 i915_vma_put(batch);
 926                 return ERR_CAST(cs);
 927         }
 928
 929         defaults = shmem_pin_map(ce->engine->default_state);
 930         if (!defaults) {
 931                 i915_gem_object_unpin_map(batch->obj);
 932                 i915_vma_put(batch);
 933                 return ERR_PTR(-ENOMEM);
 934         }
 935
 936         x = 0;
 937         dw = 0;
 938         hw = defaults;
 939         hw += LRC_STATE_OFFSET / sizeof(*hw);
 940         do {
 941                 u32 len = hw[dw] & 0x7f;
 942
 943                 if (hw[dw] == 0) {
 944                         dw++;
 945                         continue;
 946                 }
 947
 948                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 949                         dw += len + 2;
 950                         continue;
 951                 }
 952
 953                 dw++;
 954                 len = (len + 1) / 2;
 955                 while (len--) {
 956                         *cs++ = MI_STORE_REGISTER_MEM_GEN8;
 957                         *cs++ = hw[dw];
 958                         *cs++ = lower_32_bits(scratch->node.start + x);
 959                         *cs++ = upper_32_bits(scratch->node.start + x);
 960
 961                         dw += 2;
 962                         x += 4;
 963                 }
 964         } while (dw < PAGE_SIZE / sizeof(u32) &&
 965                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 966
 967         *cs++ = MI_BATCH_BUFFER_END;
 968
 969         shmem_unpin_map(ce->engine->default_state, defaults);
 970
 971         i915_gem_object_flush_map(batch->obj);
 972         i915_gem_object_unpin_map(batch->obj);
 973
 974         return batch;
 975 }
 976
 977 static int move_to_active(struct i915_request *rq,
 978                           struct i915_vma *vma,
 979                           unsigned int flags)
 980 {
 981         int err;
 982
 983         i915_vma_lock(vma);
 984         err = i915_request_await_object(rq, vma->obj, flags);
 985         if (!err)
 986                 err = i915_vma_move_to_active(vma, rq, flags);
 987         i915_vma_unlock(vma);
 988
 989         return err;
 990 }
 991
 992 static struct i915_request *
 993 record_registers(struct intel_context *ce,
 994                  struct i915_vma *before,
 995                  struct i915_vma *after,
 996                  u32 *sema)
 997 {
 998         struct i915_vma *b_before, *b_after;
 999         struct i915_request *rq;
1000         u32 *cs;
1001         int err;
1002
1003         b_before = store_context(ce, before);
1004         if (IS_ERR(b_before))
1005                 return ERR_CAST(b_before);
1006
1007         b_after = store_context(ce, after);
1008         if (IS_ERR(b_after)) {
1009                 rq = ERR_CAST(b_after);
1010                 goto err_before;
1011         }
1012
1013         rq = intel_context_create_request(ce);
1014         if (IS_ERR(rq))
1015                 goto err_after;
1016
1017         err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1018         if (err)
1019                 goto err_rq;
1020
1021         err = move_to_active(rq, b_before, 0);
1022         if (err)
1023                 goto err_rq;
1024
1025         err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1026         if (err)
1027                 goto err_rq;
1028
1029         err = move_to_active(rq, b_after, 0);
1030         if (err)
1031                 goto err_rq;
1032
1033         cs = intel_ring_begin(rq, 14);
1034         if (IS_ERR(cs)) {
1035                 err = PTR_ERR(cs);
1036                 goto err_rq;
1037         }
1038
1039         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1040         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1041         *cs++ = lower_32_bits(b_before->node.start);
1042         *cs++ = upper_32_bits(b_before->node.start);
1043
1044         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1045         *cs++ = MI_SEMAPHORE_WAIT |
1046                 MI_SEMAPHORE_GLOBAL_GTT |
1047                 MI_SEMAPHORE_POLL |
1048                 MI_SEMAPHORE_SAD_NEQ_SDD;
1049         *cs++ = 0;
1050         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1051                 offset_in_page(sema);
1052         *cs++ = 0;
1053         *cs++ = MI_NOOP;
1054
1055         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1056         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1057         *cs++ = lower_32_bits(b_after->node.start);
1058         *cs++ = upper_32_bits(b_after->node.start);
1059
1060         intel_ring_advance(rq, cs);
1061
1062         WRITE_ONCE(*sema, 0);
1063         i915_request_get(rq);
1064         i915_request_add(rq);
1065 err_after:
1066         i915_vma_put(b_after);
1067 err_before:
1068         i915_vma_put(b_before);
1069         return rq;
1070
1071 err_rq:
1072         i915_request_add(rq);
1073         rq = ERR_PTR(err);
1074         goto err_after;
1075 }
1076
1077 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1078 {
1079         struct i915_vma *batch;
1080         u32 dw, *cs, *hw;
1081         u32 *defaults;
1082
1083         batch = create_user_vma(ce->vm, SZ_64K);
1084         if (IS_ERR(batch))
1085                 return batch;
1086
1087         cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1088         if (IS_ERR(cs)) {
1089                 i915_vma_put(batch);
1090                 return ERR_CAST(cs);
1091         }
1092
1093         defaults = shmem_pin_map(ce->engine->default_state);
1094         if (!defaults) {
1095                 i915_gem_object_unpin_map(batch->obj);
1096                 i915_vma_put(batch);
1097                 return ERR_PTR(-ENOMEM);
1098         }
1099
1100         dw = 0;
1101         hw = defaults;
1102         hw += LRC_STATE_OFFSET / sizeof(*hw);
1103         do {
1104                 u32 len = hw[dw] & 0x7f;
1105
1106                 if (hw[dw] == 0) {
1107                         dw++;
1108                         continue;
1109                 }
1110
1111                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1112                         dw += len + 2;
1113                         continue;
1114                 }
1115
1116                 dw++;
1117                 len = (len + 1) / 2;
1118                 *cs++ = MI_LOAD_REGISTER_IMM(len);
1119                 while (len--) {
1120                         *cs++ = hw[dw];
1121                         *cs++ = poison;
1122                         dw += 2;
1123                 }
1124         } while (dw < PAGE_SIZE / sizeof(u32) &&
1125                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1126
1127         *cs++ = MI_BATCH_BUFFER_END;
1128
1129         shmem_unpin_map(ce->engine->default_state, defaults);
1130
1131         i915_gem_object_flush_map(batch->obj);
1132         i915_gem_object_unpin_map(batch->obj);
1133
1134         return batch;
1135 }
1136
1137 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1138 {
1139         struct i915_request *rq;
1140         struct i915_vma *batch;
1141         u32 *cs;
1142         int err;
1143
1144         batch = load_context(ce, poison);
1145         if (IS_ERR(batch))
1146                 return PTR_ERR(batch);
1147
1148         rq = intel_context_create_request(ce);
1149         if (IS_ERR(rq)) {
1150                 err = PTR_ERR(rq);
1151                 goto err_batch;
1152         }
1153
1154         err = move_to_active(rq, batch, 0);
1155         if (err)
1156                 goto err_rq;
1157
1158         cs = intel_ring_begin(rq, 8);
1159         if (IS_ERR(cs)) {
1160                 err = PTR_ERR(cs);
1161                 goto err_rq;
1162         }
1163
1164         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1165         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1166         *cs++ = lower_32_bits(batch->node.start);
1167         *cs++ = upper_32_bits(batch->node.start);
1168
1169         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1170         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1171                 offset_in_page(sema);
1172         *cs++ = 0;
1173         *cs++ = 1;
1174
1175         intel_ring_advance(rq, cs);
1176
1177         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1178 err_rq:
1179         i915_request_add(rq);
1180 err_batch:
1181         i915_vma_put(batch);
1182         return err;
1183 }
1184
1185 static bool is_moving(u32 a, u32 b)
1186 {
1187         return a != b;
1188 }
1189
1190 static int compare_isolation(struct intel_engine_cs *engine,
1191                              struct i915_vma *ref[2],
1192                              struct i915_vma *result[2],
1193                              struct intel_context *ce,
1194                              u32 poison)
1195 {
1196         u32 x, dw, *hw, *lrc;
1197         u32 *A[2], *B[2];
1198         u32 *defaults;
1199         int err = 0;
1200
1201         A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1202         if (IS_ERR(A[0]))
1203                 return PTR_ERR(A[0]);
1204
1205         A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1206         if (IS_ERR(A[1])) {
1207                 err = PTR_ERR(A[1]);
1208                 goto err_A0;
1209         }
1210
1211         B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1212         if (IS_ERR(B[0])) {
1213                 err = PTR_ERR(B[0]);
1214                 goto err_A1;
1215         }
1216
1217         B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1218         if (IS_ERR(B[1])) {
1219                 err = PTR_ERR(B[1]);
1220                 goto err_B0;
1221         }
1222
1223         lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1224                                                i915_coherent_map_type(engine->i915,
1225                                                                       ce->state->obj,
1226                                                                       false));
1227         if (IS_ERR(lrc)) {
1228                 err = PTR_ERR(lrc);
1229                 goto err_B1;
1230         }
1231         lrc += LRC_STATE_OFFSET / sizeof(*hw);
1232
1233         defaults = shmem_pin_map(ce->engine->default_state);
1234         if (!defaults) {
1235                 err = -ENOMEM;
1236                 goto err_lrc;
1237         }
1238
1239         x = 0;
1240         dw = 0;
1241         hw = defaults;
1242         hw += LRC_STATE_OFFSET / sizeof(*hw);
1243         do {
1244                 u32 len = hw[dw] & 0x7f;
1245
1246                 if (hw[dw] == 0) {
1247                         dw++;
1248                         continue;
1249                 }
1250
1251                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1252                         dw += len + 2;
1253                         continue;
1254                 }
1255
1256                 dw++;
1257                 len = (len + 1) / 2;
1258                 while (len--) {
1259                         if (!is_moving(A[0][x], A[1][x]) &&
1260                             (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1261                                 switch (hw[dw] & 4095) {
1262                                 case 0x30: /* RING_HEAD */
1263                                 case 0x34: /* RING_TAIL */
1264                                         break;
1265
1266                                 default:
1267                                         pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1268                                                engine->name, dw,
1269                                                hw[dw], hw[dw + 1],
1270                                                A[0][x], B[0][x], B[1][x],
1271                                                poison, lrc[dw + 1]);
1272                                         err = -EINVAL;
1273                                 }
1274                         }
1275                         dw += 2;
1276                         x++;
1277                 }
1278         } while (dw < PAGE_SIZE / sizeof(u32) &&
1279                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1280
1281         shmem_unpin_map(ce->engine->default_state, defaults);
1282 err_lrc:
1283         i915_gem_object_unpin_map(ce->state->obj);
1284 err_B1:
1285         i915_gem_object_unpin_map(result[1]->obj);
1286 err_B0:
1287         i915_gem_object_unpin_map(result[0]->obj);
1288 err_A1:
1289         i915_gem_object_unpin_map(ref[1]->obj);
1290 err_A0:
1291         i915_gem_object_unpin_map(ref[0]->obj);
1292         return err;
1293 }
1294
1295 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1296 {
1297         u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1298         struct i915_vma *ref[2], *result[2];
1299         struct intel_context *A, *B;
1300         struct i915_request *rq;
1301         int err;
1302
1303         A = intel_context_create(engine);
1304         if (IS_ERR(A))
1305                 return PTR_ERR(A);
1306
1307         B = intel_context_create(engine);
1308         if (IS_ERR(B)) {
1309                 err = PTR_ERR(B);
1310                 goto err_A;
1311         }
1312
1313         ref[0] = create_user_vma(A->vm, SZ_64K);
1314         if (IS_ERR(ref[0])) {
1315                 err = PTR_ERR(ref[0]);
1316                 goto err_B;
1317         }
1318
1319         ref[1] = create_user_vma(A->vm, SZ_64K);
1320         if (IS_ERR(ref[1])) {
1321                 err = PTR_ERR(ref[1]);
1322                 goto err_ref0;
1323         }
1324
1325         rq = record_registers(A, ref[0], ref[1], sema);
1326         if (IS_ERR(rq)) {
1327                 err = PTR_ERR(rq);
1328                 goto err_ref1;
1329         }
1330
1331         WRITE_ONCE(*sema, 1);
1332         wmb();
1333
1334         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1335                 i915_request_put(rq);
1336                 err = -ETIME;
1337                 goto err_ref1;
1338         }
1339         i915_request_put(rq);
1340
1341         result[0] = create_user_vma(A->vm, SZ_64K);
1342         if (IS_ERR(result[0])) {
1343                 err = PTR_ERR(result[0]);
1344                 goto err_ref1;
1345         }
1346
1347         result[1] = create_user_vma(A->vm, SZ_64K);
1348         if (IS_ERR(result[1])) {
1349                 err = PTR_ERR(result[1]);
1350                 goto err_result0;
1351         }
1352
1353         rq = record_registers(A, result[0], result[1], sema);
1354         if (IS_ERR(rq)) {
1355                 err = PTR_ERR(rq);
1356                 goto err_result1;
1357         }
1358
1359         err = poison_registers(B, poison, sema);
1360         if (err) {
1361                 WRITE_ONCE(*sema, -1);
1362                 i915_request_put(rq);
1363                 goto err_result1;
1364         }
1365
1366         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1367                 i915_request_put(rq);
1368                 err = -ETIME;
1369                 goto err_result1;
1370         }
1371         i915_request_put(rq);
1372
1373         err = compare_isolation(engine, ref, result, A, poison);
1374
1375 err_result1:
1376         i915_vma_put(result[1]);
1377 err_result0:
1378         i915_vma_put(result[0]);
1379 err_ref1:
1380         i915_vma_put(ref[1]);
1381 err_ref0:
1382         i915_vma_put(ref[0]);
1383 err_B:
1384         intel_context_put(B);
1385 err_A:
1386         intel_context_put(A);
1387         return err;
1388 }
1389
1390 static bool skip_isolation(const struct intel_engine_cs *engine)
1391 {
1392         if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1393                 return true;
1394
1395         if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1396                 return true;
1397
1398         return false;
1399 }
1400
1401 static int live_lrc_isolation(void *arg)
1402 {
1403         struct intel_gt *gt = arg;
1404         struct intel_engine_cs *engine;
1405         enum intel_engine_id id;
1406         const u32 poison[] = {
1407                 STACK_MAGIC,
1408                 0x3a3a3a3a,
1409                 0x5c5c5c5c,
1410                 0xffffffff,
1411                 0xffff0000,
1412         };
1413         int err = 0;
1414
1415         /*
1416          * Our goal is try and verify that per-context state cannot be
1417          * tampered with by another non-privileged client.
1418          *
1419          * We take the list of context registers from the LRI in the default
1420          * context image and attempt to modify that list from a remote context.
1421          */
1422
1423         for_each_engine(engine, gt, id) {
1424                 int i;
1425
1426                 /* Just don't even ask */
1427                 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1428                     skip_isolation(engine))
1429                         continue;
1430
1431                 intel_engine_pm_get(engine);
1432                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
1433                         int result;
1434
1435                         result = __lrc_isolation(engine, poison[i]);
1436                         if (result && !err)
1437                                 err = result;
1438
1439                         result = __lrc_isolation(engine, ~poison[i]);
1440                         if (result && !err)
1441                                 err = result;
1442                 }
1443                 intel_engine_pm_put(engine);
1444                 if (igt_flush_test(gt->i915)) {
1445                         err = -EIO;
1446                         break;
1447                 }
1448         }
1449
1450         return err;
1451 }
1452
1453 static int indirect_ctx_submit_req(struct intel_context *ce)
1454 {
1455         struct i915_request *rq;
1456         int err = 0;
1457
1458         rq = intel_context_create_request(ce);
1459         if (IS_ERR(rq))
1460                 return PTR_ERR(rq);
1461
1462         i915_request_get(rq);
1463         i915_request_add(rq);
1464
1465         if (i915_request_wait(rq, 0, HZ / 5) < 0)
1466                 err = -ETIME;
1467
1468         i915_request_put(rq);
1469
1470         return err;
1471 }
1472
1473 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1474 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1475
1476 static u32 *
1477 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1478 {
1479         *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1480                 MI_SRM_LRM_GLOBAL_GTT |
1481                 MI_LRI_LRM_CS_MMIO;
1482         *cs++ = i915_mmio_reg_offset(RING_START(0));
1483         *cs++ = i915_ggtt_offset(ce->state) +
1484                 context_wa_bb_offset(ce) +
1485                 CTX_BB_CANARY_OFFSET;
1486         *cs++ = 0;
1487
1488         return cs;
1489 }
1490
1491 static void
1492 indirect_ctx_bb_setup(struct intel_context *ce)
1493 {
1494         u32 *cs = context_indirect_bb(ce);
1495
1496         cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1497
1498         setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1499 }
1500
1501 static bool check_ring_start(struct intel_context *ce)
1502 {
1503         const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1504                 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1505
1506         if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1507                 return true;
1508
1509         pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1510                ctx_bb[CTX_BB_CANARY_INDEX],
1511                ce->lrc_reg_state[CTX_RING_START]);
1512
1513         return false;
1514 }
1515
1516 static int indirect_ctx_bb_check(struct intel_context *ce)
1517 {
1518         int err;
1519
1520         err = indirect_ctx_submit_req(ce);
1521         if (err)
1522                 return err;
1523
1524         if (!check_ring_start(ce))
1525                 return -EINVAL;
1526
1527         return 0;
1528 }
1529
1530 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1531 {
1532         struct intel_context *a, *b;
1533         int err;
1534
1535         a = intel_context_create(engine);
1536         if (IS_ERR(a))
1537                 return PTR_ERR(a);
1538         err = intel_context_pin(a);
1539         if (err)
1540                 goto put_a;
1541
1542         b = intel_context_create(engine);
1543         if (IS_ERR(b)) {
1544                 err = PTR_ERR(b);
1545                 goto unpin_a;
1546         }
1547         err = intel_context_pin(b);
1548         if (err)
1549                 goto put_b;
1550
1551         /* We use the already reserved extra page in context state */
1552         if (!a->wa_bb_page) {
1553                 GEM_BUG_ON(b->wa_bb_page);
1554                 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1555                 goto unpin_b;
1556         }
1557
1558         /*
1559          * In order to test that our per context bb is truly per context,
1560          * and executes at the intended spot on context restoring process,
1561          * make the batch store the ring start value to memory.
1562          * As ring start is restored apriori of starting the indirect ctx bb and
1563          * as it will be different for each context, it fits to this purpose.
1564          */
1565         indirect_ctx_bb_setup(a);
1566         indirect_ctx_bb_setup(b);
1567
1568         err = indirect_ctx_bb_check(a);
1569         if (err)
1570                 goto unpin_b;
1571
1572         err = indirect_ctx_bb_check(b);
1573
1574 unpin_b:
1575         intel_context_unpin(b);
1576 put_b:
1577         intel_context_put(b);
1578 unpin_a:
1579         intel_context_unpin(a);
1580 put_a:
1581         intel_context_put(a);
1582
1583         return err;
1584 }
1585
1586 static int live_lrc_indirect_ctx_bb(void *arg)
1587 {
1588         struct intel_gt *gt = arg;
1589         struct intel_engine_cs *engine;
1590         enum intel_engine_id id;
1591         int err = 0;
1592
1593         for_each_engine(engine, gt, id) {
1594                 intel_engine_pm_get(engine);
1595                 err = __live_lrc_indirect_ctx_bb(engine);
1596                 intel_engine_pm_put(engine);
1597
1598                 if (igt_flush_test(gt->i915))
1599                         err = -EIO;
1600
1601                 if (err)
1602                         break;
1603         }
1604
1605         return err;
1606 }
1607
1608 static void garbage_reset(struct intel_engine_cs *engine,
1609                           struct i915_request *rq)
1610 {
1611         const unsigned int bit = I915_RESET_ENGINE + engine->id;
1612         unsigned long *lock = &engine->gt->reset.flags;
1613
1614         local_bh_disable();
1615         if (!test_and_set_bit(bit, lock)) {
1616                 tasklet_disable(&engine->sched_engine->tasklet);
1617
1618                 if (!rq->fence.error)
1619                         __intel_engine_reset_bh(engine, NULL);
1620
1621                 tasklet_enable(&engine->sched_engine->tasklet);
1622                 clear_and_wake_up_bit(bit, lock);
1623         }
1624         local_bh_enable();
1625 }
1626
1627 static struct i915_request *garbage(struct intel_context *ce,
1628                                     struct rnd_state *prng)
1629 {
1630         struct i915_request *rq;
1631         int err;
1632
1633         err = intel_context_pin(ce);
1634         if (err)
1635                 return ERR_PTR(err);
1636
1637         prandom_bytes_state(prng,
1638                             ce->lrc_reg_state,
1639                             ce->engine->context_size -
1640                             LRC_STATE_OFFSET);
1641
1642         rq = intel_context_create_request(ce);
1643         if (IS_ERR(rq)) {
1644                 err = PTR_ERR(rq);
1645                 goto err_unpin;
1646         }
1647
1648         i915_request_get(rq);
1649         i915_request_add(rq);
1650         return rq;
1651
1652 err_unpin:
1653         intel_context_unpin(ce);
1654         return ERR_PTR(err);
1655 }
1656
1657 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1658 {
1659         struct intel_context *ce;
1660         struct i915_request *hang;
1661         int err = 0;
1662
1663         ce = intel_context_create(engine);
1664         if (IS_ERR(ce))
1665                 return PTR_ERR(ce);
1666
1667         hang = garbage(ce, prng);
1668         if (IS_ERR(hang)) {
1669                 err = PTR_ERR(hang);
1670                 goto err_ce;
1671         }
1672
1673         if (wait_for_submit(engine, hang, HZ / 2)) {
1674                 i915_request_put(hang);
1675                 err = -ETIME;
1676                 goto err_ce;
1677         }
1678
1679         intel_context_set_banned(ce);
1680         garbage_reset(engine, hang);
1681
1682         intel_engine_flush_submission(engine);
1683         if (!hang->fence.error) {
1684                 i915_request_put(hang);
1685                 pr_err("%s: corrupted context was not reset\n",
1686                        engine->name);
1687                 err = -EINVAL;
1688                 goto err_ce;
1689         }
1690
1691         if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1692                 pr_err("%s: corrupted context did not recover\n",
1693                        engine->name);
1694                 i915_request_put(hang);
1695                 err = -EIO;
1696                 goto err_ce;
1697         }
1698         i915_request_put(hang);
1699
1700 err_ce:
1701         intel_context_put(ce);
1702         return err;
1703 }
1704
1705 static int live_lrc_garbage(void *arg)
1706 {
1707         struct intel_gt *gt = arg;
1708         struct intel_engine_cs *engine;
1709         enum intel_engine_id id;
1710
1711         /*
1712          * Verify that we can recover if one context state is completely
1713          * corrupted.
1714          */
1715
1716         if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1717                 return 0;
1718
1719         for_each_engine(engine, gt, id) {
1720                 I915_RND_STATE(prng);
1721                 int err = 0, i;
1722
1723                 if (!intel_has_reset_engine(engine->gt))
1724                         continue;
1725
1726                 intel_engine_pm_get(engine);
1727                 for (i = 0; i < 3; i++) {
1728                         err = __lrc_garbage(engine, &prng);
1729                         if (err)
1730                                 break;
1731                 }
1732                 intel_engine_pm_put(engine);
1733
1734                 if (igt_flush_test(gt->i915))
1735                         err = -EIO;
1736                 if (err)
1737                         return err;
1738         }
1739
1740         return 0;
1741 }
1742
1743 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1744 {
1745         struct intel_context *ce;
1746         struct i915_request *rq;
1747         IGT_TIMEOUT(end_time);
1748         int err;
1749
1750         ce = intel_context_create(engine);
1751         if (IS_ERR(ce))
1752                 return PTR_ERR(ce);
1753
1754         ce->runtime.num_underflow = 0;
1755         ce->runtime.max_underflow = 0;
1756
1757         do {
1758                 unsigned int loop = 1024;
1759
1760                 while (loop) {
1761                         rq = intel_context_create_request(ce);
1762                         if (IS_ERR(rq)) {
1763                                 err = PTR_ERR(rq);
1764                                 goto err_rq;
1765                         }
1766
1767                         if (--loop == 0)
1768                                 i915_request_get(rq);
1769
1770                         i915_request_add(rq);
1771                 }
1772
1773                 if (__igt_timeout(end_time, NULL))
1774                         break;
1775
1776                 i915_request_put(rq);
1777         } while (1);
1778
1779         err = i915_request_wait(rq, 0, HZ / 5);
1780         if (err < 0) {
1781                 pr_err("%s: request not completed!\n", engine->name);
1782                 goto err_wait;
1783         }
1784
1785         igt_flush_test(engine->i915);
1786
1787         pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1788                 engine->name,
1789                 intel_context_get_total_runtime_ns(ce),
1790                 intel_context_get_avg_runtime_ns(ce));
1791
1792         err = 0;
1793         if (ce->runtime.num_underflow) {
1794                 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1795                        engine->name,
1796                        ce->runtime.num_underflow,
1797                        ce->runtime.max_underflow);
1798                 GEM_TRACE_DUMP();
1799                 err = -EOVERFLOW;
1800         }
1801
1802 err_wait:
1803         i915_request_put(rq);
1804 err_rq:
1805         intel_context_put(ce);
1806         return err;
1807 }
1808
1809 static int live_pphwsp_runtime(void *arg)
1810 {
1811         struct intel_gt *gt = arg;
1812         struct intel_engine_cs *engine;
1813         enum intel_engine_id id;
1814         int err = 0;
1815
1816         /*
1817          * Check that cumulative context runtime as stored in the pphwsp[16]
1818          * is monotonic.
1819          */
1820
1821         for_each_engine(engine, gt, id) {
1822                 err = __live_pphwsp_runtime(engine);
1823                 if (err)
1824                         break;
1825         }
1826
1827         if (igt_flush_test(gt->i915))
1828                 err = -EIO;
1829
1830         return err;
1831 }
1832
1833 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1834 {
1835         static const struct i915_subtest tests[] = {
1836                 SUBTEST(live_lrc_layout),
1837                 SUBTEST(live_lrc_fixed),
1838                 SUBTEST(live_lrc_state),
1839                 SUBTEST(live_lrc_gpr),
1840                 SUBTEST(live_lrc_isolation),
1841                 SUBTEST(live_lrc_timestamp),
1842                 SUBTEST(live_lrc_garbage),
1843                 SUBTEST(live_pphwsp_runtime),
1844                 SUBTEST(live_lrc_indirect_ctx_bb),
1845         };
1846
1847         if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1848                 return 0;
1849
1850         return intel_gt_live_subtests(tests, &i915->gt);
1851 }