drivers/gpu/drm/i915/gt/selftest_lrc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include <linux/prime_numbers.h>
   7
   8 #include "i915_selftest.h"
   9 #include "intel_engine_heartbeat.h"
  10 #include "intel_engine_pm.h"
  11 #include "intel_reset.h"
  12 #include "intel_ring.h"
  13 #include "selftest_engine_heartbeat.h"
  14 #include "selftests/i915_random.h"
  15 #include "selftests/igt_flush_test.h"
  16 #include "selftests/igt_live_test.h"
  17 #include "selftests/igt_spinner.h"
  18 #include "selftests/lib_sw_fence.h"
  19 #include "shmem_utils.h"
  20
  21 #include "gem/selftests/igt_gem_utils.h"
  22 #include "gem/selftests/mock_context.h"
  23
  24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
  25 #define NUM_GPR 16
  26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
  27
  28 static struct i915_vma *create_scratch(struct intel_gt *gt)
  29 {
  30         return __vm_create_scratch_for_read(&gt->ggtt->vm, PAGE_SIZE);
  31 }
  32
  33 static bool is_active(struct i915_request *rq)
  34 {
  35         if (i915_request_is_active(rq))
  36                 return true;
  37
  38         if (i915_request_on_hold(rq))
  39                 return true;
  40
  41         if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
  42                 return true;
  43
  44         return false;
  45 }
  46
  47 static int wait_for_submit(struct intel_engine_cs *engine,
  48                            struct i915_request *rq,
  49                            unsigned long timeout)
  50 {
  51         /* Ignore our own attempts to suppress excess tasklets */
  52         tasklet_hi_schedule(&engine->execlists.tasklet);
  53
  54         timeout += jiffies;
  55         do {
  56                 bool done = time_after(jiffies, timeout);
  57
  58                 if (i915_request_completed(rq)) /* that was quick! */
  59                         return 0;
  60
  61                 /* Wait until the HW has acknowleged the submission (or err) */
  62                 intel_engine_flush_submission(engine);
  63                 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
  64                         return 0;
  65
  66                 if (done)
  67                         return -ETIME;
  68
  69                 cond_resched();
  70         } while (1);
  71 }
  72
  73 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
  74 {
  75         const u32 offset =
  76                 i915_ggtt_offset(ce->engine->status_page.vma) +
  77                 offset_in_page(slot);
  78         struct i915_request *rq;
  79         u32 *cs;
  80
  81         rq = intel_context_create_request(ce);
  82         if (IS_ERR(rq))
  83                 return PTR_ERR(rq);
  84
  85         cs = intel_ring_begin(rq, 4);
  86         if (IS_ERR(cs)) {
  87                 i915_request_add(rq);
  88                 return PTR_ERR(cs);
  89         }
  90
  91         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  92         *cs++ = offset;
  93         *cs++ = 0;
  94         *cs++ = 1;
  95
  96         intel_ring_advance(rq, cs);
  97
  98         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
  99         i915_request_add(rq);
 100         return 0;
 101 }
 102
 103 static int context_flush(struct intel_context *ce, long timeout)
 104 {
 105         struct i915_request *rq;
 106         struct dma_fence *fence;
 107         int err = 0;
 108
 109         rq = intel_engine_create_kernel_request(ce->engine);
 110         if (IS_ERR(rq))
 111                 return PTR_ERR(rq);
 112
 113         fence = i915_active_fence_get(&ce->timeline->last_request);
 114         if (fence) {
 115                 i915_request_await_dma_fence(rq, fence);
 116                 dma_fence_put(fence);
 117         }
 118
 119         rq = i915_request_get(rq);
 120         i915_request_add(rq);
 121         if (i915_request_wait(rq, 0, timeout) < 0)
 122                 err = -ETIME;
 123         i915_request_put(rq);
 124
 125         rmb(); /* We know the request is written, make sure all state is too! */
 126         return err;
 127 }
 128
 129 static int live_lrc_layout(void *arg)
 130 {
 131         struct intel_gt *gt = arg;
 132         struct intel_engine_cs *engine;
 133         enum intel_engine_id id;
 134         u32 *lrc;
 135         int err;
 136
 137         /*
 138          * Check the registers offsets we use to create the initial reg state
 139          * match the layout saved by HW.
 140          */
 141
 142         lrc = kmalloc(PAGE_SIZE, GFP_KERNEL);
 143         if (!lrc)
 144                 return -ENOMEM;
 145
 146         err = 0;
 147         for_each_engine(engine, gt, id) {
 148                 u32 *hw;
 149                 int dw;
 150
 151                 if (!engine->default_state)
 152                         continue;
 153
 154                 hw = shmem_pin_map(engine->default_state);
 155                 if (IS_ERR(hw)) {
 156                         err = PTR_ERR(hw);
 157                         break;
 158                 }
 159                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 160
 161                 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
 162                                 engine->kernel_context, engine, true);
 163
 164                 dw = 0;
 165                 do {
 166                         u32 lri = hw[dw];
 167
 168                         if (lri == 0) {
 169                                 dw++;
 170                                 continue;
 171                         }
 172
 173                         if (lrc[dw] == 0) {
 174                                 pr_debug("%s: skipped instruction %x at dword %d\n",
 175                                          engine->name, lri, dw);
 176                                 dw++;
 177                                 continue;
 178                         }
 179
 180                         if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 181                                 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
 182                                        engine->name, dw, lri);
 183                                 err = -EINVAL;
 184                                 break;
 185                         }
 186
 187                         if (lrc[dw] != lri) {
 188                                 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
 189                                        engine->name, dw, lri, lrc[dw]);
 190                                 err = -EINVAL;
 191                                 break;
 192                         }
 193
 194                         lri &= 0x7f;
 195                         lri++;
 196                         dw++;
 197
 198                         while (lri) {
 199                                 if (hw[dw] != lrc[dw]) {
 200                                         pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
 201                                                engine->name, dw, hw[dw], lrc[dw]);
 202                                         err = -EINVAL;
 203                                         break;
 204                                 }
 205
 206                                 /*
 207                                  * Skip over the actual register value as we
 208                                  * expect that to differ.
 209                                  */
 210                                 dw += 2;
 211                                 lri -= 2;
 212                         }
 213                 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 214
 215                 if (err) {
 216                         pr_info("%s: HW register image:\n", engine->name);
 217                         igt_hexdump(hw, PAGE_SIZE);
 218
 219                         pr_info("%s: SW register image:\n", engine->name);
 220                         igt_hexdump(lrc, PAGE_SIZE);
 221                 }
 222
 223                 shmem_unpin_map(engine->default_state, hw);
 224                 if (err)
 225                         break;
 226         }
 227
 228         kfree(lrc);
 229         return err;
 230 }
 231
 232 static int find_offset(const u32 *lri, u32 offset)
 233 {
 234         int i;
 235
 236         for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
 237                 if (lri[i] == offset)
 238                         return i;
 239
 240         return -1;
 241 }
 242
 243 static int live_lrc_fixed(void *arg)
 244 {
 245         struct intel_gt *gt = arg;
 246         struct intel_engine_cs *engine;
 247         enum intel_engine_id id;
 248         int err = 0;
 249
 250         /*
 251          * Check the assumed register offsets match the actual locations in
 252          * the context image.
 253          */
 254
 255         for_each_engine(engine, gt, id) {
 256                 const struct {
 257                         u32 reg;
 258                         u32 offset;
 259                         const char *name;
 260                 } tbl[] = {
 261                         {
 262                                 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
 263                                 CTX_RING_START - 1,
 264                                 "RING_START"
 265                         },
 266                         {
 267                                 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
 268                                 CTX_RING_CTL - 1,
 269                                 "RING_CTL"
 270                         },
 271                         {
 272                                 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
 273                                 CTX_RING_HEAD - 1,
 274                                 "RING_HEAD"
 275                         },
 276                         {
 277                                 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
 278                                 CTX_RING_TAIL - 1,
 279                                 "RING_TAIL"
 280                         },
 281                         {
 282                                 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
 283                                 lrc_ring_mi_mode(engine),
 284                                 "RING_MI_MODE"
 285                         },
 286                         {
 287                                 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
 288                                 CTX_BB_STATE - 1,
 289                                 "BB_STATE"
 290                         },
 291                         {
 292                                 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
 293                                 lrc_ring_wa_bb_per_ctx(engine),
 294                                 "RING_BB_PER_CTX_PTR"
 295                         },
 296                         {
 297                                 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
 298                                 lrc_ring_indirect_ptr(engine),
 299                                 "RING_INDIRECT_CTX_PTR"
 300                         },
 301                         {
 302                                 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
 303                                 lrc_ring_indirect_offset(engine),
 304                                 "RING_INDIRECT_CTX_OFFSET"
 305                         },
 306                         {
 307                                 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
 308                                 CTX_TIMESTAMP - 1,
 309                                 "RING_CTX_TIMESTAMP"
 310                         },
 311                         {
 312                                 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
 313                                 lrc_ring_gpr0(engine),
 314                                 "RING_CS_GPR0"
 315                         },
 316                         {
 317                                 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
 318                                 lrc_ring_cmd_buf_cctl(engine),
 319                                 "RING_CMD_BUF_CCTL"
 320                         },
 321                         { },
 322                 }, *t;
 323                 u32 *hw;
 324
 325                 if (!engine->default_state)
 326                         continue;
 327
 328                 hw = shmem_pin_map(engine->default_state);
 329                 if (IS_ERR(hw)) {
 330                         err = PTR_ERR(hw);
 331                         break;
 332                 }
 333                 hw += LRC_STATE_OFFSET / sizeof(*hw);
 334
 335                 for (t = tbl; t->name; t++) {
 336                         int dw = find_offset(hw, t->reg);
 337
 338                         if (dw != t->offset) {
 339                                 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
 340                                        engine->name,
 341                                        t->name,
 342                                        t->reg,
 343                                        dw,
 344                                        t->offset);
 345                                 err = -EINVAL;
 346                         }
 347                 }
 348
 349                 shmem_unpin_map(engine->default_state, hw);
 350         }
 351
 352         return err;
 353 }
 354
 355 static int __live_lrc_state(struct intel_engine_cs *engine,
 356                             struct i915_vma *scratch)
 357 {
 358         struct intel_context *ce;
 359         struct i915_request *rq;
 360         struct i915_gem_ww_ctx ww;
 361         enum {
 362                 RING_START_IDX = 0,
 363                 RING_TAIL_IDX,
 364                 MAX_IDX
 365         };
 366         u32 expected[MAX_IDX];
 367         u32 *cs;
 368         int err;
 369         int n;
 370
 371         ce = intel_context_create(engine);
 372         if (IS_ERR(ce))
 373                 return PTR_ERR(ce);
 374
 375         i915_gem_ww_ctx_init(&ww, false);
 376 retry:
 377         err = i915_gem_object_lock(scratch->obj, &ww);
 378         if (!err)
 379                 err = intel_context_pin_ww(ce, &ww);
 380         if (err)
 381                 goto err_put;
 382
 383         rq = i915_request_create(ce);
 384         if (IS_ERR(rq)) {
 385                 err = PTR_ERR(rq);
 386                 goto err_unpin;
 387         }
 388
 389         cs = intel_ring_begin(rq, 4 * MAX_IDX);
 390         if (IS_ERR(cs)) {
 391                 err = PTR_ERR(cs);
 392                 i915_request_add(rq);
 393                 goto err_unpin;
 394         }
 395
 396         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 397         *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
 398         *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
 399         *cs++ = 0;
 400
 401         expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
 402
 403         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 404         *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
 405         *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
 406         *cs++ = 0;
 407
 408         err = i915_request_await_object(rq, scratch->obj, true);
 409         if (!err)
 410                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 411
 412         i915_request_get(rq);
 413         i915_request_add(rq);
 414         if (err)
 415                 goto err_rq;
 416
 417         intel_engine_flush_submission(engine);
 418         expected[RING_TAIL_IDX] = ce->ring->tail;
 419
 420         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 421                 err = -ETIME;
 422                 goto err_rq;
 423         }
 424
 425         cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 426         if (IS_ERR(cs)) {
 427                 err = PTR_ERR(cs);
 428                 goto err_rq;
 429         }
 430
 431         for (n = 0; n < MAX_IDX; n++) {
 432                 if (cs[n] != expected[n]) {
 433                         pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
 434                                engine->name, n, cs[n], expected[n]);
 435                         err = -EINVAL;
 436                         break;
 437                 }
 438         }
 439
 440         i915_gem_object_unpin_map(scratch->obj);
 441
 442 err_rq:
 443         i915_request_put(rq);
 444 err_unpin:
 445         intel_context_unpin(ce);
 446 err_put:
 447         if (err == -EDEADLK) {
 448                 err = i915_gem_ww_ctx_backoff(&ww);
 449                 if (!err)
 450                         goto retry;
 451         }
 452         i915_gem_ww_ctx_fini(&ww);
 453         intel_context_put(ce);
 454         return err;
 455 }
 456
 457 static int live_lrc_state(void *arg)
 458 {
 459         struct intel_gt *gt = arg;
 460         struct intel_engine_cs *engine;
 461         struct i915_vma *scratch;
 462         enum intel_engine_id id;
 463         int err = 0;
 464
 465         /*
 466          * Check the live register state matches what we expect for this
 467          * intel_context.
 468          */
 469
 470         scratch = create_scratch(gt);
 471         if (IS_ERR(scratch))
 472                 return PTR_ERR(scratch);
 473
 474         for_each_engine(engine, gt, id) {
 475                 err = __live_lrc_state(engine, scratch);
 476                 if (err)
 477                         break;
 478         }
 479
 480         if (igt_flush_test(gt->i915))
 481                 err = -EIO;
 482
 483         i915_vma_unpin_and_release(&scratch, 0);
 484         return err;
 485 }
 486
 487 static int gpr_make_dirty(struct intel_context *ce)
 488 {
 489         struct i915_request *rq;
 490         u32 *cs;
 491         int n;
 492
 493         rq = intel_context_create_request(ce);
 494         if (IS_ERR(rq))
 495                 return PTR_ERR(rq);
 496
 497         cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
 498         if (IS_ERR(cs)) {
 499                 i915_request_add(rq);
 500                 return PTR_ERR(cs);
 501         }
 502
 503         *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
 504         for (n = 0; n < NUM_GPR_DW; n++) {
 505                 *cs++ = CS_GPR(ce->engine, n);
 506                 *cs++ = STACK_MAGIC;
 507         }
 508         *cs++ = MI_NOOP;
 509
 510         intel_ring_advance(rq, cs);
 511
 512         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
 513         i915_request_add(rq);
 514
 515         return 0;
 516 }
 517
 518 static struct i915_request *
 519 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
 520 {
 521         const u32 offset =
 522                 i915_ggtt_offset(ce->engine->status_page.vma) +
 523                 offset_in_page(slot);
 524         struct i915_request *rq;
 525         u32 *cs;
 526         int err;
 527         int n;
 528
 529         rq = intel_context_create_request(ce);
 530         if (IS_ERR(rq))
 531                 return rq;
 532
 533         cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
 534         if (IS_ERR(cs)) {
 535                 i915_request_add(rq);
 536                 return ERR_CAST(cs);
 537         }
 538
 539         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 540         *cs++ = MI_NOOP;
 541
 542         *cs++ = MI_SEMAPHORE_WAIT |
 543                 MI_SEMAPHORE_GLOBAL_GTT |
 544                 MI_SEMAPHORE_POLL |
 545                 MI_SEMAPHORE_SAD_NEQ_SDD;
 546         *cs++ = 0;
 547         *cs++ = offset;
 548         *cs++ = 0;
 549
 550         for (n = 0; n < NUM_GPR_DW; n++) {
 551                 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 552                 *cs++ = CS_GPR(ce->engine, n);
 553                 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
 554                 *cs++ = 0;
 555         }
 556
 557         i915_vma_lock(scratch);
 558         err = i915_request_await_object(rq, scratch->obj, true);
 559         if (!err)
 560                 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 561         i915_vma_unlock(scratch);
 562
 563         i915_request_get(rq);
 564         i915_request_add(rq);
 565         if (err) {
 566                 i915_request_put(rq);
 567                 rq = ERR_PTR(err);
 568         }
 569
 570         return rq;
 571 }
 572
 573 static int __live_lrc_gpr(struct intel_engine_cs *engine,
 574                           struct i915_vma *scratch,
 575                           bool preempt)
 576 {
 577         u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
 578         struct intel_context *ce;
 579         struct i915_request *rq;
 580         u32 *cs;
 581         int err;
 582         int n;
 583
 584         if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
 585                 return 0; /* GPR only on rcs0 for gen8 */
 586
 587         err = gpr_make_dirty(engine->kernel_context);
 588         if (err)
 589                 return err;
 590
 591         ce = intel_context_create(engine);
 592         if (IS_ERR(ce))
 593                 return PTR_ERR(ce);
 594
 595         rq = __gpr_read(ce, scratch, slot);
 596         if (IS_ERR(rq)) {
 597                 err = PTR_ERR(rq);
 598                 goto err_put;
 599         }
 600
 601         err = wait_for_submit(engine, rq, HZ / 2);
 602         if (err)
 603                 goto err_rq;
 604
 605         if (preempt) {
 606                 err = gpr_make_dirty(engine->kernel_context);
 607                 if (err)
 608                         goto err_rq;
 609
 610                 err = emit_semaphore_signal(engine->kernel_context, slot);
 611                 if (err)
 612                         goto err_rq;
 613
 614                 err = wait_for_submit(engine, rq, HZ / 2);
 615                 if (err)
 616                         goto err_rq;
 617         } else {
 618                 slot[0] = 1;
 619                 wmb();
 620         }
 621
 622         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 623                 err = -ETIME;
 624                 goto err_rq;
 625         }
 626
 627         cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 628         if (IS_ERR(cs)) {
 629                 err = PTR_ERR(cs);
 630                 goto err_rq;
 631         }
 632
 633         for (n = 0; n < NUM_GPR_DW; n++) {
 634                 if (cs[n]) {
 635                         pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
 636                                engine->name,
 637                                n / 2, n & 1 ? "udw" : "ldw",
 638                                cs[n]);
 639                         err = -EINVAL;
 640                         break;
 641                 }
 642         }
 643
 644         i915_gem_object_unpin_map(scratch->obj);
 645
 646 err_rq:
 647         memset32(&slot[0], -1, 4);
 648         wmb();
 649         i915_request_put(rq);
 650 err_put:
 651         intel_context_put(ce);
 652         return err;
 653 }
 654
 655 static int live_lrc_gpr(void *arg)
 656 {
 657         struct intel_gt *gt = arg;
 658         struct intel_engine_cs *engine;
 659         struct i915_vma *scratch;
 660         enum intel_engine_id id;
 661         int err = 0;
 662
 663         /*
 664          * Check that GPR registers are cleared in new contexts as we need
 665          * to avoid leaking any information from previous contexts.
 666          */
 667
 668         scratch = create_scratch(gt);
 669         if (IS_ERR(scratch))
 670                 return PTR_ERR(scratch);
 671
 672         for_each_engine(engine, gt, id) {
 673                 st_engine_heartbeat_disable(engine);
 674
 675                 err = __live_lrc_gpr(engine, scratch, false);
 676                 if (err)
 677                         goto err;
 678
 679                 err = __live_lrc_gpr(engine, scratch, true);
 680                 if (err)
 681                         goto err;
 682
 683 err:
 684                 st_engine_heartbeat_enable(engine);
 685                 if (igt_flush_test(gt->i915))
 686                         err = -EIO;
 687                 if (err)
 688                         break;
 689         }
 690
 691         i915_vma_unpin_and_release(&scratch, 0);
 692         return err;
 693 }
 694
 695 static struct i915_request *
 696 create_timestamp(struct intel_context *ce, void *slot, int idx)
 697 {
 698         const u32 offset =
 699                 i915_ggtt_offset(ce->engine->status_page.vma) +
 700                 offset_in_page(slot);
 701         struct i915_request *rq;
 702         u32 *cs;
 703         int err;
 704
 705         rq = intel_context_create_request(ce);
 706         if (IS_ERR(rq))
 707                 return rq;
 708
 709         cs = intel_ring_begin(rq, 10);
 710         if (IS_ERR(cs)) {
 711                 err = PTR_ERR(cs);
 712                 goto err;
 713         }
 714
 715         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 716         *cs++ = MI_NOOP;
 717
 718         *cs++ = MI_SEMAPHORE_WAIT |
 719                 MI_SEMAPHORE_GLOBAL_GTT |
 720                 MI_SEMAPHORE_POLL |
 721                 MI_SEMAPHORE_SAD_NEQ_SDD;
 722         *cs++ = 0;
 723         *cs++ = offset;
 724         *cs++ = 0;
 725
 726         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 727         *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
 728         *cs++ = offset + idx * sizeof(u32);
 729         *cs++ = 0;
 730
 731         intel_ring_advance(rq, cs);
 732
 733         rq->sched.attr.priority = I915_PRIORITY_MASK;
 734         err = 0;
 735 err:
 736         i915_request_get(rq);
 737         i915_request_add(rq);
 738         if (err) {
 739                 i915_request_put(rq);
 740                 return ERR_PTR(err);
 741         }
 742
 743         return rq;
 744 }
 745
 746 struct lrc_timestamp {
 747         struct intel_engine_cs *engine;
 748         struct intel_context *ce[2];
 749         u32 poison;
 750 };
 751
 752 static bool timestamp_advanced(u32 start, u32 end)
 753 {
 754         return (s32)(end - start) > 0;
 755 }
 756
 757 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
 758 {
 759         u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
 760         struct i915_request *rq;
 761         u32 timestamp;
 762         int err = 0;
 763
 764         arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
 765         rq = create_timestamp(arg->ce[0], slot, 1);
 766         if (IS_ERR(rq))
 767                 return PTR_ERR(rq);
 768
 769         err = wait_for_submit(rq->engine, rq, HZ / 2);
 770         if (err)
 771                 goto err;
 772
 773         if (preempt) {
 774                 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
 775                 err = emit_semaphore_signal(arg->ce[1], slot);
 776                 if (err)
 777                         goto err;
 778         } else {
 779                 slot[0] = 1;
 780                 wmb();
 781         }
 782
 783         /* And wait for switch to kernel (to save our context to memory) */
 784         err = context_flush(arg->ce[0], HZ / 2);
 785         if (err)
 786                 goto err;
 787
 788         if (!timestamp_advanced(arg->poison, slot[1])) {
 789                 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
 790                        arg->engine->name, preempt ? "preempt" : "simple",
 791                        arg->poison, slot[1]);
 792                 err = -EINVAL;
 793         }
 794
 795         timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
 796         if (!timestamp_advanced(slot[1], timestamp)) {
 797                 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
 798                        arg->engine->name, preempt ? "preempt" : "simple",
 799                        slot[1], timestamp);
 800                 err = -EINVAL;
 801         }
 802
 803 err:
 804         memset32(slot, -1, 4);
 805         i915_request_put(rq);
 806         return err;
 807 }
 808
 809 static int live_lrc_timestamp(void *arg)
 810 {
 811         struct lrc_timestamp data = {};
 812         struct intel_gt *gt = arg;
 813         enum intel_engine_id id;
 814         const u32 poison[] = {
 815                 0,
 816                 S32_MAX,
 817                 (u32)S32_MAX + 1,
 818                 U32_MAX,
 819         };
 820
 821         /*
 822          * We want to verify that the timestamp is saved and restore across
 823          * context switches and is monotonic.
 824          *
 825          * So we do this with a little bit of LRC poisoning to check various
 826          * boundary conditions, and see what happens if we preempt the context
 827          * with a second request (carrying more poison into the timestamp).
 828          */
 829
 830         for_each_engine(data.engine, gt, id) {
 831                 int i, err = 0;
 832
 833                 st_engine_heartbeat_disable(data.engine);
 834
 835                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 836                         struct intel_context *tmp;
 837
 838                         tmp = intel_context_create(data.engine);
 839                         if (IS_ERR(tmp)) {
 840                                 err = PTR_ERR(tmp);
 841                                 goto err;
 842                         }
 843
 844                         err = intel_context_pin(tmp);
 845                         if (err) {
 846                                 intel_context_put(tmp);
 847                                 goto err;
 848                         }
 849
 850                         data.ce[i] = tmp;
 851                 }
 852
 853                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
 854                         data.poison = poison[i];
 855
 856                         err = __lrc_timestamp(&data, false);
 857                         if (err)
 858                                 break;
 859
 860                         err = __lrc_timestamp(&data, true);
 861                         if (err)
 862                                 break;
 863                 }
 864
 865 err:
 866                 st_engine_heartbeat_enable(data.engine);
 867                 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 868                         if (!data.ce[i])
 869                                 break;
 870
 871                         intel_context_unpin(data.ce[i]);
 872                         intel_context_put(data.ce[i]);
 873                 }
 874
 875                 if (igt_flush_test(gt->i915))
 876                         err = -EIO;
 877                 if (err)
 878                         return err;
 879         }
 880
 881         return 0;
 882 }
 883
 884 static struct i915_vma *
 885 create_user_vma(struct i915_address_space *vm, unsigned long size)
 886 {
 887         struct drm_i915_gem_object *obj;
 888         struct i915_vma *vma;
 889         int err;
 890
 891         obj = i915_gem_object_create_internal(vm->i915, size);
 892         if (IS_ERR(obj))
 893                 return ERR_CAST(obj);
 894
 895         vma = i915_vma_instance(obj, vm, NULL);
 896         if (IS_ERR(vma)) {
 897                 i915_gem_object_put(obj);
 898                 return vma;
 899         }
 900
 901         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 902         if (err) {
 903                 i915_gem_object_put(obj);
 904                 return ERR_PTR(err);
 905         }
 906
 907         return vma;
 908 }
 909
 910 static struct i915_vma *
 911 store_context(struct intel_context *ce, struct i915_vma *scratch)
 912 {
 913         struct i915_vma *batch;
 914         u32 dw, x, *cs, *hw;
 915         u32 *defaults;
 916
 917         batch = create_user_vma(ce->vm, SZ_64K);
 918         if (IS_ERR(batch))
 919                 return batch;
 920
 921         cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 922         if (IS_ERR(cs)) {
 923                 i915_vma_put(batch);
 924                 return ERR_CAST(cs);
 925         }
 926
 927         defaults = shmem_pin_map(ce->engine->default_state);
 928         if (!defaults) {
 929                 i915_gem_object_unpin_map(batch->obj);
 930                 i915_vma_put(batch);
 931                 return ERR_PTR(-ENOMEM);
 932         }
 933
 934         x = 0;
 935         dw = 0;
 936         hw = defaults;
 937         hw += LRC_STATE_OFFSET / sizeof(*hw);
 938         do {
 939                 u32 len = hw[dw] & 0x7f;
 940
 941                 if (hw[dw] == 0) {
 942                         dw++;
 943                         continue;
 944                 }
 945
 946                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 947                         dw += len + 2;
 948                         continue;
 949                 }
 950
 951                 dw++;
 952                 len = (len + 1) / 2;
 953                 while (len--) {
 954                         *cs++ = MI_STORE_REGISTER_MEM_GEN8;
 955                         *cs++ = hw[dw];
 956                         *cs++ = lower_32_bits(scratch->node.start + x);
 957                         *cs++ = upper_32_bits(scratch->node.start + x);
 958
 959                         dw += 2;
 960                         x += 4;
 961                 }
 962         } while (dw < PAGE_SIZE / sizeof(u32) &&
 963                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 964
 965         *cs++ = MI_BATCH_BUFFER_END;
 966
 967         shmem_unpin_map(ce->engine->default_state, defaults);
 968
 969         i915_gem_object_flush_map(batch->obj);
 970         i915_gem_object_unpin_map(batch->obj);
 971
 972         return batch;
 973 }
 974
 975 static int move_to_active(struct i915_request *rq,
 976                           struct i915_vma *vma,
 977                           unsigned int flags)
 978 {
 979         int err;
 980
 981         i915_vma_lock(vma);
 982         err = i915_request_await_object(rq, vma->obj, flags);
 983         if (!err)
 984                 err = i915_vma_move_to_active(vma, rq, flags);
 985         i915_vma_unlock(vma);
 986
 987         return err;
 988 }
 989
 990 static struct i915_request *
 991 record_registers(struct intel_context *ce,
 992                  struct i915_vma *before,
 993                  struct i915_vma *after,
 994                  u32 *sema)
 995 {
 996         struct i915_vma *b_before, *b_after;
 997         struct i915_request *rq;
 998         u32 *cs;
 999         int err;
1000
1001         b_before = store_context(ce, before);
1002         if (IS_ERR(b_before))
1003                 return ERR_CAST(b_before);
1004
1005         b_after = store_context(ce, after);
1006         if (IS_ERR(b_after)) {
1007                 rq = ERR_CAST(b_after);
1008                 goto err_before;
1009         }
1010
1011         rq = intel_context_create_request(ce);
1012         if (IS_ERR(rq))
1013                 goto err_after;
1014
1015         err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1016         if (err)
1017                 goto err_rq;
1018
1019         err = move_to_active(rq, b_before, 0);
1020         if (err)
1021                 goto err_rq;
1022
1023         err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1024         if (err)
1025                 goto err_rq;
1026
1027         err = move_to_active(rq, b_after, 0);
1028         if (err)
1029                 goto err_rq;
1030
1031         cs = intel_ring_begin(rq, 14);
1032         if (IS_ERR(cs)) {
1033                 err = PTR_ERR(cs);
1034                 goto err_rq;
1035         }
1036
1037         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1038         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1039         *cs++ = lower_32_bits(b_before->node.start);
1040         *cs++ = upper_32_bits(b_before->node.start);
1041
1042         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1043         *cs++ = MI_SEMAPHORE_WAIT |
1044                 MI_SEMAPHORE_GLOBAL_GTT |
1045                 MI_SEMAPHORE_POLL |
1046                 MI_SEMAPHORE_SAD_NEQ_SDD;
1047         *cs++ = 0;
1048         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1049                 offset_in_page(sema);
1050         *cs++ = 0;
1051         *cs++ = MI_NOOP;
1052
1053         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1054         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1055         *cs++ = lower_32_bits(b_after->node.start);
1056         *cs++ = upper_32_bits(b_after->node.start);
1057
1058         intel_ring_advance(rq, cs);
1059
1060         WRITE_ONCE(*sema, 0);
1061         i915_request_get(rq);
1062         i915_request_add(rq);
1063 err_after:
1064         i915_vma_put(b_after);
1065 err_before:
1066         i915_vma_put(b_before);
1067         return rq;
1068
1069 err_rq:
1070         i915_request_add(rq);
1071         rq = ERR_PTR(err);
1072         goto err_after;
1073 }
1074
1075 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1076 {
1077         struct i915_vma *batch;
1078         u32 dw, *cs, *hw;
1079         u32 *defaults;
1080
1081         batch = create_user_vma(ce->vm, SZ_64K);
1082         if (IS_ERR(batch))
1083                 return batch;
1084
1085         cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
1086         if (IS_ERR(cs)) {
1087                 i915_vma_put(batch);
1088                 return ERR_CAST(cs);
1089         }
1090
1091         defaults = shmem_pin_map(ce->engine->default_state);
1092         if (!defaults) {
1093                 i915_gem_object_unpin_map(batch->obj);
1094                 i915_vma_put(batch);
1095                 return ERR_PTR(-ENOMEM);
1096         }
1097
1098         dw = 0;
1099         hw = defaults;
1100         hw += LRC_STATE_OFFSET / sizeof(*hw);
1101         do {
1102                 u32 len = hw[dw] & 0x7f;
1103
1104                 if (hw[dw] == 0) {
1105                         dw++;
1106                         continue;
1107                 }
1108
1109                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1110                         dw += len + 2;
1111                         continue;
1112                 }
1113
1114                 dw++;
1115                 len = (len + 1) / 2;
1116                 *cs++ = MI_LOAD_REGISTER_IMM(len);
1117                 while (len--) {
1118                         *cs++ = hw[dw];
1119                         *cs++ = poison;
1120                         dw += 2;
1121                 }
1122         } while (dw < PAGE_SIZE / sizeof(u32) &&
1123                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1124
1125         *cs++ = MI_BATCH_BUFFER_END;
1126
1127         shmem_unpin_map(ce->engine->default_state, defaults);
1128
1129         i915_gem_object_flush_map(batch->obj);
1130         i915_gem_object_unpin_map(batch->obj);
1131
1132         return batch;
1133 }
1134
1135 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1136 {
1137         struct i915_request *rq;
1138         struct i915_vma *batch;
1139         u32 *cs;
1140         int err;
1141
1142         batch = load_context(ce, poison);
1143         if (IS_ERR(batch))
1144                 return PTR_ERR(batch);
1145
1146         rq = intel_context_create_request(ce);
1147         if (IS_ERR(rq)) {
1148                 err = PTR_ERR(rq);
1149                 goto err_batch;
1150         }
1151
1152         err = move_to_active(rq, batch, 0);
1153         if (err)
1154                 goto err_rq;
1155
1156         cs = intel_ring_begin(rq, 8);
1157         if (IS_ERR(cs)) {
1158                 err = PTR_ERR(cs);
1159                 goto err_rq;
1160         }
1161
1162         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1163         *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1164         *cs++ = lower_32_bits(batch->node.start);
1165         *cs++ = upper_32_bits(batch->node.start);
1166
1167         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1168         *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1169                 offset_in_page(sema);
1170         *cs++ = 0;
1171         *cs++ = 1;
1172
1173         intel_ring_advance(rq, cs);
1174
1175         rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1176 err_rq:
1177         i915_request_add(rq);
1178 err_batch:
1179         i915_vma_put(batch);
1180         return err;
1181 }
1182
1183 static bool is_moving(u32 a, u32 b)
1184 {
1185         return a != b;
1186 }
1187
1188 static int compare_isolation(struct intel_engine_cs *engine,
1189                              struct i915_vma *ref[2],
1190                              struct i915_vma *result[2],
1191                              struct intel_context *ce,
1192                              u32 poison)
1193 {
1194         u32 x, dw, *hw, *lrc;
1195         u32 *A[2], *B[2];
1196         u32 *defaults;
1197         int err = 0;
1198
1199         A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);
1200         if (IS_ERR(A[0]))
1201                 return PTR_ERR(A[0]);
1202
1203         A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);
1204         if (IS_ERR(A[1])) {
1205                 err = PTR_ERR(A[1]);
1206                 goto err_A0;
1207         }
1208
1209         B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);
1210         if (IS_ERR(B[0])) {
1211                 err = PTR_ERR(B[0]);
1212                 goto err_A1;
1213         }
1214
1215         B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);
1216         if (IS_ERR(B[1])) {
1217                 err = PTR_ERR(B[1]);
1218                 goto err_B0;
1219         }
1220
1221         lrc = i915_gem_object_pin_map(ce->state->obj,
1222                                       i915_coherent_map_type(engine->i915));
1223         if (IS_ERR(lrc)) {
1224                 err = PTR_ERR(lrc);
1225                 goto err_B1;
1226         }
1227         lrc += LRC_STATE_OFFSET / sizeof(*hw);
1228
1229         defaults = shmem_pin_map(ce->engine->default_state);
1230         if (!defaults) {
1231                 err = -ENOMEM;
1232                 goto err_lrc;
1233         }
1234
1235         x = 0;
1236         dw = 0;
1237         hw = defaults;
1238         hw += LRC_STATE_OFFSET / sizeof(*hw);
1239         do {
1240                 u32 len = hw[dw] & 0x7f;
1241
1242                 if (hw[dw] == 0) {
1243                         dw++;
1244                         continue;
1245                 }
1246
1247                 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1248                         dw += len + 2;
1249                         continue;
1250                 }
1251
1252                 dw++;
1253                 len = (len + 1) / 2;
1254                 while (len--) {
1255                         if (!is_moving(A[0][x], A[1][x]) &&
1256                             (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1257                                 switch (hw[dw] & 4095) {
1258                                 case 0x30: /* RING_HEAD */
1259                                 case 0x34: /* RING_TAIL */
1260                                         break;
1261
1262                                 default:
1263                                         pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1264                                                engine->name, dw,
1265                                                hw[dw], hw[dw + 1],
1266                                                A[0][x], B[0][x], B[1][x],
1267                                                poison, lrc[dw + 1]);
1268                                         err = -EINVAL;
1269                                 }
1270                         }
1271                         dw += 2;
1272                         x++;
1273                 }
1274         } while (dw < PAGE_SIZE / sizeof(u32) &&
1275                  (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1276
1277         shmem_unpin_map(ce->engine->default_state, defaults);
1278 err_lrc:
1279         i915_gem_object_unpin_map(ce->state->obj);
1280 err_B1:
1281         i915_gem_object_unpin_map(result[1]->obj);
1282 err_B0:
1283         i915_gem_object_unpin_map(result[0]->obj);
1284 err_A1:
1285         i915_gem_object_unpin_map(ref[1]->obj);
1286 err_A0:
1287         i915_gem_object_unpin_map(ref[0]->obj);
1288         return err;
1289 }
1290
1291 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1292 {
1293         u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1294         struct i915_vma *ref[2], *result[2];
1295         struct intel_context *A, *B;
1296         struct i915_request *rq;
1297         int err;
1298
1299         A = intel_context_create(engine);
1300         if (IS_ERR(A))
1301                 return PTR_ERR(A);
1302
1303         B = intel_context_create(engine);
1304         if (IS_ERR(B)) {
1305                 err = PTR_ERR(B);
1306                 goto err_A;
1307         }
1308
1309         ref[0] = create_user_vma(A->vm, SZ_64K);
1310         if (IS_ERR(ref[0])) {
1311                 err = PTR_ERR(ref[0]);
1312                 goto err_B;
1313         }
1314
1315         ref[1] = create_user_vma(A->vm, SZ_64K);
1316         if (IS_ERR(ref[1])) {
1317                 err = PTR_ERR(ref[1]);
1318                 goto err_ref0;
1319         }
1320
1321         rq = record_registers(A, ref[0], ref[1], sema);
1322         if (IS_ERR(rq)) {
1323                 err = PTR_ERR(rq);
1324                 goto err_ref1;
1325         }
1326
1327         WRITE_ONCE(*sema, 1);
1328         wmb();
1329
1330         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1331                 i915_request_put(rq);
1332                 err = -ETIME;
1333                 goto err_ref1;
1334         }
1335         i915_request_put(rq);
1336
1337         result[0] = create_user_vma(A->vm, SZ_64K);
1338         if (IS_ERR(result[0])) {
1339                 err = PTR_ERR(result[0]);
1340                 goto err_ref1;
1341         }
1342
1343         result[1] = create_user_vma(A->vm, SZ_64K);
1344         if (IS_ERR(result[1])) {
1345                 err = PTR_ERR(result[1]);
1346                 goto err_result0;
1347         }
1348
1349         rq = record_registers(A, result[0], result[1], sema);
1350         if (IS_ERR(rq)) {
1351                 err = PTR_ERR(rq);
1352                 goto err_result1;
1353         }
1354
1355         err = poison_registers(B, poison, sema);
1356         if (err) {
1357                 WRITE_ONCE(*sema, -1);
1358                 i915_request_put(rq);
1359                 goto err_result1;
1360         }
1361
1362         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1363                 i915_request_put(rq);
1364                 err = -ETIME;
1365                 goto err_result1;
1366         }
1367         i915_request_put(rq);
1368
1369         err = compare_isolation(engine, ref, result, A, poison);
1370
1371 err_result1:
1372         i915_vma_put(result[1]);
1373 err_result0:
1374         i915_vma_put(result[0]);
1375 err_ref1:
1376         i915_vma_put(ref[1]);
1377 err_ref0:
1378         i915_vma_put(ref[0]);
1379 err_B:
1380         intel_context_put(B);
1381 err_A:
1382         intel_context_put(A);
1383         return err;
1384 }
1385
1386 static bool skip_isolation(const struct intel_engine_cs *engine)
1387 {
1388         if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
1389                 return true;
1390
1391         if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
1392                 return true;
1393
1394         return false;
1395 }
1396
1397 static int live_lrc_isolation(void *arg)
1398 {
1399         struct intel_gt *gt = arg;
1400         struct intel_engine_cs *engine;
1401         enum intel_engine_id id;
1402         const u32 poison[] = {
1403                 STACK_MAGIC,
1404                 0x3a3a3a3a,
1405                 0x5c5c5c5c,
1406                 0xffffffff,
1407                 0xffff0000,
1408         };
1409         int err = 0;
1410
1411         /*
1412          * Our goal is try and verify that per-context state cannot be
1413          * tampered with by another non-privileged client.
1414          *
1415          * We take the list of context registers from the LRI in the default
1416          * context image and attempt to modify that list from a remote context.
1417          */
1418
1419         for_each_engine(engine, gt, id) {
1420                 int i;
1421
1422                 /* Just don't even ask */
1423                 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1424                     skip_isolation(engine))
1425                         continue;
1426
1427                 intel_engine_pm_get(engine);
1428                 for (i = 0; i < ARRAY_SIZE(poison); i++) {
1429                         int result;
1430
1431                         result = __lrc_isolation(engine, poison[i]);
1432                         if (result && !err)
1433                                 err = result;
1434
1435                         result = __lrc_isolation(engine, ~poison[i]);
1436                         if (result && !err)
1437                                 err = result;
1438                 }
1439                 intel_engine_pm_put(engine);
1440                 if (igt_flush_test(gt->i915)) {
1441                         err = -EIO;
1442                         break;
1443                 }
1444         }
1445
1446         return err;
1447 }
1448
1449 static int indirect_ctx_submit_req(struct intel_context *ce)
1450 {
1451         struct i915_request *rq;
1452         int err = 0;
1453
1454         rq = intel_context_create_request(ce);
1455         if (IS_ERR(rq))
1456                 return PTR_ERR(rq);
1457
1458         i915_request_get(rq);
1459         i915_request_add(rq);
1460
1461         if (i915_request_wait(rq, 0, HZ / 5) < 0)
1462                 err = -ETIME;
1463
1464         i915_request_put(rq);
1465
1466         return err;
1467 }
1468
1469 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1470 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1471
1472 static u32 *
1473 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1474 {
1475         *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1476                 MI_SRM_LRM_GLOBAL_GTT |
1477                 MI_LRI_LRM_CS_MMIO;
1478         *cs++ = i915_mmio_reg_offset(RING_START(0));
1479         *cs++ = i915_ggtt_offset(ce->state) +
1480                 context_wa_bb_offset(ce) +
1481                 CTX_BB_CANARY_OFFSET;
1482         *cs++ = 0;
1483
1484         return cs;
1485 }
1486
1487 static void
1488 indirect_ctx_bb_setup(struct intel_context *ce)
1489 {
1490         u32 *cs = context_indirect_bb(ce);
1491
1492         cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1493
1494         setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1495 }
1496
1497 static bool check_ring_start(struct intel_context *ce)
1498 {
1499         const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1500                 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1501
1502         if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1503                 return true;
1504
1505         pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1506                ctx_bb[CTX_BB_CANARY_INDEX],
1507                ce->lrc_reg_state[CTX_RING_START]);
1508
1509         return false;
1510 }
1511
1512 static int indirect_ctx_bb_check(struct intel_context *ce)
1513 {
1514         int err;
1515
1516         err = indirect_ctx_submit_req(ce);
1517         if (err)
1518                 return err;
1519
1520         if (!check_ring_start(ce))
1521                 return -EINVAL;
1522
1523         return 0;
1524 }
1525
1526 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1527 {
1528         struct intel_context *a, *b;
1529         int err;
1530
1531         a = intel_context_create(engine);
1532         if (IS_ERR(a))
1533                 return PTR_ERR(a);
1534         err = intel_context_pin(a);
1535         if (err)
1536                 goto put_a;
1537
1538         b = intel_context_create(engine);
1539         if (IS_ERR(b)) {
1540                 err = PTR_ERR(b);
1541                 goto unpin_a;
1542         }
1543         err = intel_context_pin(b);
1544         if (err)
1545                 goto put_b;
1546
1547         /* We use the already reserved extra page in context state */
1548         if (!a->wa_bb_page) {
1549                 GEM_BUG_ON(b->wa_bb_page);
1550                 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12);
1551                 goto unpin_b;
1552         }
1553
1554         /*
1555          * In order to test that our per context bb is truly per context,
1556          * and executes at the intended spot on context restoring process,
1557          * make the batch store the ring start value to memory.
1558          * As ring start is restored apriori of starting the indirect ctx bb and
1559          * as it will be different for each context, it fits to this purpose.
1560          */
1561         indirect_ctx_bb_setup(a);
1562         indirect_ctx_bb_setup(b);
1563
1564         err = indirect_ctx_bb_check(a);
1565         if (err)
1566                 goto unpin_b;
1567
1568         err = indirect_ctx_bb_check(b);
1569
1570 unpin_b:
1571         intel_context_unpin(b);
1572 put_b:
1573         intel_context_put(b);
1574 unpin_a:
1575         intel_context_unpin(a);
1576 put_a:
1577         intel_context_put(a);
1578
1579         return err;
1580 }
1581
1582 static int live_lrc_indirect_ctx_bb(void *arg)
1583 {
1584         struct intel_gt *gt = arg;
1585         struct intel_engine_cs *engine;
1586         enum intel_engine_id id;
1587         int err = 0;
1588
1589         for_each_engine(engine, gt, id) {
1590                 intel_engine_pm_get(engine);
1591                 err = __live_lrc_indirect_ctx_bb(engine);
1592                 intel_engine_pm_put(engine);
1593
1594                 if (igt_flush_test(gt->i915))
1595                         err = -EIO;
1596
1597                 if (err)
1598                         break;
1599         }
1600
1601         return err;
1602 }
1603
1604 static void garbage_reset(struct intel_engine_cs *engine,
1605                           struct i915_request *rq)
1606 {
1607         const unsigned int bit = I915_RESET_ENGINE + engine->id;
1608         unsigned long *lock = &engine->gt->reset.flags;
1609
1610         local_bh_disable();
1611         if (!test_and_set_bit(bit, lock)) {
1612                 tasklet_disable(&engine->execlists.tasklet);
1613
1614                 if (!rq->fence.error)
1615                         __intel_engine_reset_bh(engine, NULL);
1616
1617                 tasklet_enable(&engine->execlists.tasklet);
1618                 clear_and_wake_up_bit(bit, lock);
1619         }
1620         local_bh_enable();
1621 }
1622
1623 static struct i915_request *garbage(struct intel_context *ce,
1624                                     struct rnd_state *prng)
1625 {
1626         struct i915_request *rq;
1627         int err;
1628
1629         err = intel_context_pin(ce);
1630         if (err)
1631                 return ERR_PTR(err);
1632
1633         prandom_bytes_state(prng,
1634                             ce->lrc_reg_state,
1635                             ce->engine->context_size -
1636                             LRC_STATE_OFFSET);
1637
1638         rq = intel_context_create_request(ce);
1639         if (IS_ERR(rq)) {
1640                 err = PTR_ERR(rq);
1641                 goto err_unpin;
1642         }
1643
1644         i915_request_get(rq);
1645         i915_request_add(rq);
1646         return rq;
1647
1648 err_unpin:
1649         intel_context_unpin(ce);
1650         return ERR_PTR(err);
1651 }
1652
1653 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1654 {
1655         struct intel_context *ce;
1656         struct i915_request *hang;
1657         int err = 0;
1658
1659         ce = intel_context_create(engine);
1660         if (IS_ERR(ce))
1661                 return PTR_ERR(ce);
1662
1663         hang = garbage(ce, prng);
1664         if (IS_ERR(hang)) {
1665                 err = PTR_ERR(hang);
1666                 goto err_ce;
1667         }
1668
1669         if (wait_for_submit(engine, hang, HZ / 2)) {
1670                 i915_request_put(hang);
1671                 err = -ETIME;
1672                 goto err_ce;
1673         }
1674
1675         intel_context_set_banned(ce);
1676         garbage_reset(engine, hang);
1677
1678         intel_engine_flush_submission(engine);
1679         if (!hang->fence.error) {
1680                 i915_request_put(hang);
1681                 pr_err("%s: corrupted context was not reset\n",
1682                        engine->name);
1683                 err = -EINVAL;
1684                 goto err_ce;
1685         }
1686
1687         if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1688                 pr_err("%s: corrupted context did not recover\n",
1689                        engine->name);
1690                 i915_request_put(hang);
1691                 err = -EIO;
1692                 goto err_ce;
1693         }
1694         i915_request_put(hang);
1695
1696 err_ce:
1697         intel_context_put(ce);
1698         return err;
1699 }
1700
1701 static int live_lrc_garbage(void *arg)
1702 {
1703         struct intel_gt *gt = arg;
1704         struct intel_engine_cs *engine;
1705         enum intel_engine_id id;
1706
1707         /*
1708          * Verify that we can recover if one context state is completely
1709          * corrupted.
1710          */
1711
1712         if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1713                 return 0;
1714
1715         for_each_engine(engine, gt, id) {
1716                 I915_RND_STATE(prng);
1717                 int err = 0, i;
1718
1719                 if (!intel_has_reset_engine(engine->gt))
1720                         continue;
1721
1722                 intel_engine_pm_get(engine);
1723                 for (i = 0; i < 3; i++) {
1724                         err = __lrc_garbage(engine, &prng);
1725                         if (err)
1726                                 break;
1727                 }
1728                 intel_engine_pm_put(engine);
1729
1730                 if (igt_flush_test(gt->i915))
1731                         err = -EIO;
1732                 if (err)
1733                         return err;
1734         }
1735
1736         return 0;
1737 }
1738
1739 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1740 {
1741         struct intel_context *ce;
1742         struct i915_request *rq;
1743         IGT_TIMEOUT(end_time);
1744         int err;
1745
1746         ce = intel_context_create(engine);
1747         if (IS_ERR(ce))
1748                 return PTR_ERR(ce);
1749
1750         ce->runtime.num_underflow = 0;
1751         ce->runtime.max_underflow = 0;
1752
1753         do {
1754                 unsigned int loop = 1024;
1755
1756                 while (loop) {
1757                         rq = intel_context_create_request(ce);
1758                         if (IS_ERR(rq)) {
1759                                 err = PTR_ERR(rq);
1760                                 goto err_rq;
1761                         }
1762
1763                         if (--loop == 0)
1764                                 i915_request_get(rq);
1765
1766                         i915_request_add(rq);
1767                 }
1768
1769                 if (__igt_timeout(end_time, NULL))
1770                         break;
1771
1772                 i915_request_put(rq);
1773         } while (1);
1774
1775         err = i915_request_wait(rq, 0, HZ / 5);
1776         if (err < 0) {
1777                 pr_err("%s: request not completed!\n", engine->name);
1778                 goto err_wait;
1779         }
1780
1781         igt_flush_test(engine->i915);
1782
1783         pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1784                 engine->name,
1785                 intel_context_get_total_runtime_ns(ce),
1786                 intel_context_get_avg_runtime_ns(ce));
1787
1788         err = 0;
1789         if (ce->runtime.num_underflow) {
1790                 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1791                        engine->name,
1792                        ce->runtime.num_underflow,
1793                        ce->runtime.max_underflow);
1794                 GEM_TRACE_DUMP();
1795                 err = -EOVERFLOW;
1796         }
1797
1798 err_wait:
1799         i915_request_put(rq);
1800 err_rq:
1801         intel_context_put(ce);
1802         return err;
1803 }
1804
1805 static int live_pphwsp_runtime(void *arg)
1806 {
1807         struct intel_gt *gt = arg;
1808         struct intel_engine_cs *engine;
1809         enum intel_engine_id id;
1810         int err = 0;
1811
1812         /*
1813          * Check that cumulative context runtime as stored in the pphwsp[16]
1814          * is monotonic.
1815          */
1816
1817         for_each_engine(engine, gt, id) {
1818                 err = __live_pphwsp_runtime(engine);
1819                 if (err)
1820                         break;
1821         }
1822
1823         if (igt_flush_test(gt->i915))
1824                 err = -EIO;
1825
1826         return err;
1827 }
1828
1829 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1830 {
1831         static const struct i915_subtest tests[] = {
1832                 SUBTEST(live_lrc_layout),
1833                 SUBTEST(live_lrc_fixed),
1834                 SUBTEST(live_lrc_state),
1835                 SUBTEST(live_lrc_gpr),
1836                 SUBTEST(live_lrc_isolation),
1837                 SUBTEST(live_lrc_timestamp),
1838                 SUBTEST(live_lrc_garbage),
1839                 SUBTEST(live_pphwsp_runtime),
1840                 SUBTEST(live_lrc_indirect_ctx_bb),
1841         };
1842
1843         if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1844                 return 0;
1845
1846         return intel_gt_live_subtests(tests, &i915->gt);
1847 }