drivers/gpu/drm/i915/gt/selftest_hangcheck.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2016 Intel Corporation
   4  */
   5
   6 #include <linux/kthread.h>
   7
   8 #include "gem/i915_gem_context.h"
   9
  10 #include "intel_gt.h"
  11 #include "intel_engine_heartbeat.h"
  12 #include "intel_engine_pm.h"
  13 #include "selftest_engine_heartbeat.h"
  14
  15 #include "i915_selftest.h"
  16 #include "selftests/i915_random.h"
  17 #include "selftests/igt_flush_test.h"
  18 #include "selftests/igt_reset.h"
  19 #include "selftests/igt_atomic.h"
  20
  21 #include "selftests/mock_drm.h"
  22
  23 #include "gem/selftests/mock_context.h"
  24 #include "gem/selftests/igt_gem_utils.h"
  25
  26 #define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */
  27
  28 struct hang {
  29         struct intel_gt *gt;
  30         struct drm_i915_gem_object *hws;
  31         struct drm_i915_gem_object *obj;
  32         struct i915_gem_context *ctx;
  33         u32 *seqno;
  34         u32 *batch;
  35 };
  36
  37 static int hang_init(struct hang *h, struct intel_gt *gt)
  38 {
  39         void *vaddr;
  40         int err;
  41
  42         memset(h, 0, sizeof(*h));
  43         h->gt = gt;
  44
  45         h->ctx = kernel_context(gt->i915);
  46         if (IS_ERR(h->ctx))
  47                 return PTR_ERR(h->ctx);
  48
  49         GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx));
  50
  51         h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
  52         if (IS_ERR(h->hws)) {
  53                 err = PTR_ERR(h->hws);
  54                 goto err_ctx;
  55         }
  56
  57         h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
  58         if (IS_ERR(h->obj)) {
  59                 err = PTR_ERR(h->obj);
  60                 goto err_hws;
  61         }
  62
  63         i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
  64         vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);
  65         if (IS_ERR(vaddr)) {
  66                 err = PTR_ERR(vaddr);
  67                 goto err_obj;
  68         }
  69         h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
  70
  71         vaddr = i915_gem_object_pin_map_unlocked(h->obj,
  72                                                  i915_coherent_map_type(gt->i915, h->obj, false));
  73         if (IS_ERR(vaddr)) {
  74                 err = PTR_ERR(vaddr);
  75                 goto err_unpin_hws;
  76         }
  77         h->batch = vaddr;
  78
  79         return 0;
  80
  81 err_unpin_hws:
  82         i915_gem_object_unpin_map(h->hws);
  83 err_obj:
  84         i915_gem_object_put(h->obj);
  85 err_hws:
  86         i915_gem_object_put(h->hws);
  87 err_ctx:
  88         kernel_context_close(h->ctx);
  89         return err;
  90 }
  91
  92 static u64 hws_address(const struct i915_vma *hws,
  93                        const struct i915_request *rq)
  94 {
  95         return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);
  96 }
  97
  98 static int move_to_active(struct i915_vma *vma,
  99                           struct i915_request *rq,
 100                           unsigned int flags)
 101 {
 102         int err;
 103
 104         i915_vma_lock(vma);
 105         err = i915_request_await_object(rq, vma->obj,
 106                                         flags & EXEC_OBJECT_WRITE);
 107         if (err == 0)
 108                 err = i915_vma_move_to_active(vma, rq, flags);
 109         i915_vma_unlock(vma);
 110
 111         return err;
 112 }
 113
 114 static struct i915_request *
 115 hang_create_request(struct hang *h, struct intel_engine_cs *engine)
 116 {
 117         struct intel_gt *gt = h->gt;
 118         struct i915_address_space *vm = i915_gem_context_get_vm_rcu(h->ctx);
 119         struct drm_i915_gem_object *obj;
 120         struct i915_request *rq = NULL;
 121         struct i915_vma *hws, *vma;
 122         unsigned int flags;
 123         void *vaddr;
 124         u32 *batch;
 125         int err;
 126
 127         obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
 128         if (IS_ERR(obj)) {
 129                 i915_vm_put(vm);
 130                 return ERR_CAST(obj);
 131         }
 132
 133         vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false));
 134         if (IS_ERR(vaddr)) {
 135                 i915_gem_object_put(obj);
 136                 i915_vm_put(vm);
 137                 return ERR_CAST(vaddr);
 138         }
 139
 140         i915_gem_object_unpin_map(h->obj);
 141         i915_gem_object_put(h->obj);
 142
 143         h->obj = obj;
 144         h->batch = vaddr;
 145
 146         vma = i915_vma_instance(h->obj, vm, NULL);
 147         if (IS_ERR(vma)) {
 148                 i915_vm_put(vm);
 149                 return ERR_CAST(vma);
 150         }
 151
 152         hws = i915_vma_instance(h->hws, vm, NULL);
 153         if (IS_ERR(hws)) {
 154                 i915_vm_put(vm);
 155                 return ERR_CAST(hws);
 156         }
 157
 158         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 159         if (err) {
 160                 i915_vm_put(vm);
 161                 return ERR_PTR(err);
 162         }
 163
 164         err = i915_vma_pin(hws, 0, 0, PIN_USER);
 165         if (err)
 166                 goto unpin_vma;
 167
 168         rq = igt_request_alloc(h->ctx, engine);
 169         if (IS_ERR(rq)) {
 170                 err = PTR_ERR(rq);
 171                 goto unpin_hws;
 172         }
 173
 174         err = move_to_active(vma, rq, 0);
 175         if (err)
 176                 goto cancel_rq;
 177
 178         err = move_to_active(hws, rq, 0);
 179         if (err)
 180                 goto cancel_rq;
 181
 182         batch = h->batch;
 183         if (GRAPHICS_VER(gt->i915) >= 8) {
 184                 *batch++ = MI_STORE_DWORD_IMM_GEN4;
 185                 *batch++ = lower_32_bits(hws_address(hws, rq));
 186                 *batch++ = upper_32_bits(hws_address(hws, rq));
 187                 *batch++ = rq->fence.seqno;
 188                 *batch++ = MI_NOOP;
 189
 190                 memset(batch, 0, 1024);
 191                 batch += 1024 / sizeof(*batch);
 192
 193                 *batch++ = MI_NOOP;
 194                 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
 195                 *batch++ = lower_32_bits(vma->node.start);
 196                 *batch++ = upper_32_bits(vma->node.start);
 197         } else if (GRAPHICS_VER(gt->i915) >= 6) {
 198                 *batch++ = MI_STORE_DWORD_IMM_GEN4;
 199                 *batch++ = 0;
 200                 *batch++ = lower_32_bits(hws_address(hws, rq));
 201                 *batch++ = rq->fence.seqno;
 202                 *batch++ = MI_NOOP;
 203
 204                 memset(batch, 0, 1024);
 205                 batch += 1024 / sizeof(*batch);
 206
 207                 *batch++ = MI_NOOP;
 208                 *batch++ = MI_BATCH_BUFFER_START | 1 << 8;
 209                 *batch++ = lower_32_bits(vma->node.start);
 210         } else if (GRAPHICS_VER(gt->i915) >= 4) {
 211                 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 212                 *batch++ = 0;
 213                 *batch++ = lower_32_bits(hws_address(hws, rq));
 214                 *batch++ = rq->fence.seqno;
 215                 *batch++ = MI_NOOP;
 216
 217                 memset(batch, 0, 1024);
 218                 batch += 1024 / sizeof(*batch);
 219
 220                 *batch++ = MI_NOOP;
 221                 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 222                 *batch++ = lower_32_bits(vma->node.start);
 223         } else {
 224                 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 225                 *batch++ = lower_32_bits(hws_address(hws, rq));
 226                 *batch++ = rq->fence.seqno;
 227                 *batch++ = MI_NOOP;
 228
 229                 memset(batch, 0, 1024);
 230                 batch += 1024 / sizeof(*batch);
 231
 232                 *batch++ = MI_NOOP;
 233                 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 234                 *batch++ = lower_32_bits(vma->node.start);
 235         }
 236         *batch++ = MI_BATCH_BUFFER_END; /* not reached */
 237         intel_gt_chipset_flush(engine->gt);
 238
 239         if (rq->engine->emit_init_breadcrumb) {
 240                 err = rq->engine->emit_init_breadcrumb(rq);
 241                 if (err)
 242                         goto cancel_rq;
 243         }
 244
 245         flags = 0;
 246         if (GRAPHICS_VER(gt->i915) <= 5)
 247                 flags |= I915_DISPATCH_SECURE;
 248
 249         err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
 250
 251 cancel_rq:
 252         if (err) {
 253                 i915_request_set_error_once(rq, err);
 254                 i915_request_add(rq);
 255         }
 256 unpin_hws:
 257         i915_vma_unpin(hws);
 258 unpin_vma:
 259         i915_vma_unpin(vma);
 260         i915_vm_put(vm);
 261         return err ? ERR_PTR(err) : rq;
 262 }
 263
 264 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
 265 {
 266         return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
 267 }
 268
 269 static void hang_fini(struct hang *h)
 270 {
 271         *h->batch = MI_BATCH_BUFFER_END;
 272         intel_gt_chipset_flush(h->gt);
 273
 274         i915_gem_object_unpin_map(h->obj);
 275         i915_gem_object_put(h->obj);
 276
 277         i915_gem_object_unpin_map(h->hws);
 278         i915_gem_object_put(h->hws);
 279
 280         kernel_context_close(h->ctx);
 281
 282         igt_flush_test(h->gt->i915);
 283 }
 284
 285 static bool wait_until_running(struct hang *h, struct i915_request *rq)
 286 {
 287         return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
 288                                                rq->fence.seqno),
 289                              10) &&
 290                  wait_for(i915_seqno_passed(hws_seqno(h, rq),
 291                                             rq->fence.seqno),
 292                           1000));
 293 }
 294
 295 static int igt_hang_sanitycheck(void *arg)
 296 {
 297         struct intel_gt *gt = arg;
 298         struct i915_request *rq;
 299         struct intel_engine_cs *engine;
 300         enum intel_engine_id id;
 301         struct hang h;
 302         int err;
 303
 304         /* Basic check that we can execute our hanging batch */
 305
 306         err = hang_init(&h, gt);
 307         if (err)
 308                 return err;
 309
 310         for_each_engine(engine, gt, id) {
 311                 struct intel_wedge_me w;
 312                 long timeout;
 313
 314                 if (!intel_engine_can_store_dword(engine))
 315                         continue;
 316
 317                 rq = hang_create_request(&h, engine);
 318                 if (IS_ERR(rq)) {
 319                         err = PTR_ERR(rq);
 320                         pr_err("Failed to create request for %s, err=%d\n",
 321                                engine->name, err);
 322                         goto fini;
 323                 }
 324
 325                 i915_request_get(rq);
 326
 327                 *h.batch = MI_BATCH_BUFFER_END;
 328                 intel_gt_chipset_flush(engine->gt);
 329
 330                 i915_request_add(rq);
 331
 332                 timeout = 0;
 333                 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
 334                         timeout = i915_request_wait(rq, 0,
 335                                                     MAX_SCHEDULE_TIMEOUT);
 336                 if (intel_gt_is_wedged(gt))
 337                         timeout = -EIO;
 338
 339                 i915_request_put(rq);
 340
 341                 if (timeout < 0) {
 342                         err = timeout;
 343                         pr_err("Wait for request failed on %s, err=%d\n",
 344                                engine->name, err);
 345                         goto fini;
 346                 }
 347         }
 348
 349 fini:
 350         hang_fini(&h);
 351         return err;
 352 }
 353
 354 static bool wait_for_idle(struct intel_engine_cs *engine)
 355 {
 356         return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
 357 }
 358
 359 static int igt_reset_nop(void *arg)
 360 {
 361         struct intel_gt *gt = arg;
 362         struct i915_gpu_error *global = &gt->i915->gpu_error;
 363         struct intel_engine_cs *engine;
 364         unsigned int reset_count, count;
 365         enum intel_engine_id id;
 366         IGT_TIMEOUT(end_time);
 367         int err = 0;
 368
 369         /* Check that we can reset during non-user portions of requests */
 370
 371         reset_count = i915_reset_count(global);
 372         count = 0;
 373         do {
 374                 for_each_engine(engine, gt, id) {
 375                         struct intel_context *ce;
 376                         int i;
 377
 378                         ce = intel_context_create(engine);
 379                         if (IS_ERR(ce)) {
 380                                 err = PTR_ERR(ce);
 381                                 break;
 382                         }
 383
 384                         for (i = 0; i < 16; i++) {
 385                                 struct i915_request *rq;
 386
 387                                 rq = intel_context_create_request(ce);
 388                                 if (IS_ERR(rq)) {
 389                                         err = PTR_ERR(rq);
 390                                         break;
 391                                 }
 392
 393                                 i915_request_add(rq);
 394                         }
 395
 396                         intel_context_put(ce);
 397                 }
 398
 399                 igt_global_reset_lock(gt);
 400                 intel_gt_reset(gt, ALL_ENGINES, NULL);
 401                 igt_global_reset_unlock(gt);
 402
 403                 if (intel_gt_is_wedged(gt)) {
 404                         err = -EIO;
 405                         break;
 406                 }
 407
 408                 if (i915_reset_count(global) != reset_count + ++count) {
 409                         pr_err("Full GPU reset not recorded!\n");
 410                         err = -EINVAL;
 411                         break;
 412                 }
 413
 414                 err = igt_flush_test(gt->i915);
 415                 if (err)
 416                         break;
 417         } while (time_before(jiffies, end_time));
 418         pr_info("%s: %d resets\n", __func__, count);
 419
 420         if (igt_flush_test(gt->i915))
 421                 err = -EIO;
 422         return err;
 423 }
 424
 425 static int igt_reset_nop_engine(void *arg)
 426 {
 427         struct intel_gt *gt = arg;
 428         struct i915_gpu_error *global = &gt->i915->gpu_error;
 429         struct intel_engine_cs *engine;
 430         enum intel_engine_id id;
 431
 432         /* Check that we can engine-reset during non-user portions */
 433
 434         if (!intel_has_reset_engine(gt))
 435                 return 0;
 436
 437         for_each_engine(engine, gt, id) {
 438                 unsigned int reset_count, reset_engine_count, count;
 439                 struct intel_context *ce;
 440                 IGT_TIMEOUT(end_time);
 441                 int err;
 442
 443                 ce = intel_context_create(engine);
 444                 if (IS_ERR(ce))
 445                         return PTR_ERR(ce);
 446
 447                 reset_count = i915_reset_count(global);
 448                 reset_engine_count = i915_reset_engine_count(global, engine);
 449                 count = 0;
 450
 451                 st_engine_heartbeat_disable(engine);
 452                 set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 453                 do {
 454                         int i;
 455
 456                         if (!wait_for_idle(engine)) {
 457                                 pr_err("%s failed to idle before reset\n",
 458                                        engine->name);
 459                                 err = -EIO;
 460                                 break;
 461                         }
 462
 463                         for (i = 0; i < 16; i++) {
 464                                 struct i915_request *rq;
 465
 466                                 rq = intel_context_create_request(ce);
 467                                 if (IS_ERR(rq)) {
 468                                         struct drm_printer p =
 469                                                 drm_info_printer(gt->i915->drm.dev);
 470                                         intel_engine_dump(engine, &p,
 471                                                           "%s(%s): failed to submit request\n",
 472                                                           __func__,
 473                                                           engine->name);
 474
 475                                         GEM_TRACE("%s(%s): failed to submit request\n",
 476                                                   __func__,
 477                                                   engine->name);
 478                                         GEM_TRACE_DUMP();
 479
 480                                         intel_gt_set_wedged(gt);
 481
 482                                         err = PTR_ERR(rq);
 483                                         break;
 484                                 }
 485
 486                                 i915_request_add(rq);
 487                         }
 488                         err = intel_engine_reset(engine, NULL);
 489                         if (err) {
 490                                 pr_err("intel_engine_reset(%s) failed, err:%d\n",
 491                                        engine->name, err);
 492                                 break;
 493                         }
 494
 495                         if (i915_reset_count(global) != reset_count) {
 496                                 pr_err("Full GPU reset recorded! (engine reset expected)\n");
 497                                 err = -EINVAL;
 498                                 break;
 499                         }
 500
 501                         if (i915_reset_engine_count(global, engine) !=
 502                             reset_engine_count + ++count) {
 503                                 pr_err("%s engine reset not recorded!\n",
 504                                        engine->name);
 505                                 err = -EINVAL;
 506                                 break;
 507                         }
 508                 } while (time_before(jiffies, end_time));
 509                 clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 510                 st_engine_heartbeat_enable(engine);
 511
 512                 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
 513
 514                 intel_context_put(ce);
 515                 if (igt_flush_test(gt->i915))
 516                         err = -EIO;
 517                 if (err)
 518                         return err;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static void force_reset_timeout(struct intel_engine_cs *engine)
 525 {
 526         engine->reset_timeout.probability = 999;
 527         atomic_set(&engine->reset_timeout.times, -1);
 528 }
 529
 530 static void cancel_reset_timeout(struct intel_engine_cs *engine)
 531 {
 532         memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
 533 }
 534
 535 static int igt_reset_fail_engine(void *arg)
 536 {
 537         struct intel_gt *gt = arg;
 538         struct intel_engine_cs *engine;
 539         enum intel_engine_id id;
 540
 541         /* Check that we can recover from engine-reset failues */
 542
 543         if (!intel_has_reset_engine(gt))
 544                 return 0;
 545
 546         for_each_engine(engine, gt, id) {
 547                 unsigned int count;
 548                 struct intel_context *ce;
 549                 IGT_TIMEOUT(end_time);
 550                 int err;
 551
 552                 ce = intel_context_create(engine);
 553                 if (IS_ERR(ce))
 554                         return PTR_ERR(ce);
 555
 556                 st_engine_heartbeat_disable(engine);
 557                 set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 558
 559                 force_reset_timeout(engine);
 560                 err = intel_engine_reset(engine, NULL);
 561                 cancel_reset_timeout(engine);
 562                 if (err == 0) /* timeouts only generated on gen8+ */
 563                         goto skip;
 564
 565                 count = 0;
 566                 do {
 567                         struct i915_request *last = NULL;
 568                         int i;
 569
 570                         if (!wait_for_idle(engine)) {
 571                                 pr_err("%s failed to idle before reset\n",
 572                                        engine->name);
 573                                 err = -EIO;
 574                                 break;
 575                         }
 576
 577                         for (i = 0; i < count % 15; i++) {
 578                                 struct i915_request *rq;
 579
 580                                 rq = intel_context_create_request(ce);
 581                                 if (IS_ERR(rq)) {
 582                                         struct drm_printer p =
 583                                                 drm_info_printer(gt->i915->drm.dev);
 584                                         intel_engine_dump(engine, &p,
 585                                                           "%s(%s): failed to submit request\n",
 586                                                           __func__,
 587                                                           engine->name);
 588
 589                                         GEM_TRACE("%s(%s): failed to submit request\n",
 590                                                   __func__,
 591                                                   engine->name);
 592                                         GEM_TRACE_DUMP();
 593
 594                                         intel_gt_set_wedged(gt);
 595                                         if (last)
 596                                                 i915_request_put(last);
 597
 598                                         err = PTR_ERR(rq);
 599                                         goto out;
 600                                 }
 601
 602                                 if (last)
 603                                         i915_request_put(last);
 604                                 last = i915_request_get(rq);
 605                                 i915_request_add(rq);
 606                         }
 607
 608                         if (count & 1) {
 609                                 err = intel_engine_reset(engine, NULL);
 610                                 if (err) {
 611                                         GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n",
 612                                                       engine->name, err);
 613                                         GEM_TRACE_DUMP();
 614                                         i915_request_put(last);
 615                                         break;
 616                                 }
 617                         } else {
 618                                 force_reset_timeout(engine);
 619                                 err = intel_engine_reset(engine, NULL);
 620                                 cancel_reset_timeout(engine);
 621                                 if (err != -ETIMEDOUT) {
 622                                         pr_err("intel_engine_reset(%s) did not fail, err:%d\n",
 623                                                engine->name, err);
 624                                         i915_request_put(last);
 625                                         break;
 626                                 }
 627                         }
 628
 629                         err = 0;
 630                         if (last) {
 631                                 if (i915_request_wait(last, 0, HZ / 2) < 0) {
 632                                         struct drm_printer p =
 633                                                 drm_info_printer(gt->i915->drm.dev);
 634
 635                                         intel_engine_dump(engine, &p,
 636                                                           "%s(%s): failed to complete request\n",
 637                                                           __func__,
 638                                                           engine->name);
 639
 640                                         GEM_TRACE("%s(%s): failed to complete request\n",
 641                                                   __func__,
 642                                                   engine->name);
 643                                         GEM_TRACE_DUMP();
 644
 645                                         err = -EIO;
 646                                 }
 647                                 i915_request_put(last);
 648                         }
 649                         count++;
 650                 } while (err == 0 && time_before(jiffies, end_time));
 651 out:
 652                 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
 653 skip:
 654                 clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 655                 st_engine_heartbeat_enable(engine);
 656                 intel_context_put(ce);
 657
 658                 if (igt_flush_test(gt->i915))
 659                         err = -EIO;
 660                 if (err)
 661                         return err;
 662         }
 663
 664         return 0;
 665 }
 666
 667 static int __igt_reset_engine(struct intel_gt *gt, bool active)
 668 {
 669         struct i915_gpu_error *global = &gt->i915->gpu_error;
 670         struct intel_engine_cs *engine;
 671         enum intel_engine_id id;
 672         struct hang h;
 673         int err = 0;
 674
 675         /* Check that we can issue an engine reset on an idle engine (no-op) */
 676
 677         if (!intel_has_reset_engine(gt))
 678                 return 0;
 679
 680         if (active) {
 681                 err = hang_init(&h, gt);
 682                 if (err)
 683                         return err;
 684         }
 685
 686         for_each_engine(engine, gt, id) {
 687                 unsigned int reset_count, reset_engine_count;
 688                 unsigned long count;
 689                 IGT_TIMEOUT(end_time);
 690
 691                 if (active && !intel_engine_can_store_dword(engine))
 692                         continue;
 693
 694                 if (!wait_for_idle(engine)) {
 695                         pr_err("%s failed to idle before reset\n",
 696                                engine->name);
 697                         err = -EIO;
 698                         break;
 699                 }
 700
 701                 reset_count = i915_reset_count(global);
 702                 reset_engine_count = i915_reset_engine_count(global, engine);
 703
 704                 st_engine_heartbeat_disable(engine);
 705                 set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 706                 count = 0;
 707                 do {
 708                         if (active) {
 709                                 struct i915_request *rq;
 710
 711                                 rq = hang_create_request(&h, engine);
 712                                 if (IS_ERR(rq)) {
 713                                         err = PTR_ERR(rq);
 714                                         break;
 715                                 }
 716
 717                                 i915_request_get(rq);
 718                                 i915_request_add(rq);
 719
 720                                 if (!wait_until_running(&h, rq)) {
 721                                         struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
 722
 723                                         pr_err("%s: Failed to start request %llx, at %x\n",
 724                                                __func__, rq->fence.seqno, hws_seqno(&h, rq));
 725                                         intel_engine_dump(engine, &p,
 726                                                           "%s\n", engine->name);
 727
 728                                         i915_request_put(rq);
 729                                         err = -EIO;
 730                                         break;
 731                                 }
 732
 733                                 i915_request_put(rq);
 734                         }
 735
 736                         err = intel_engine_reset(engine, NULL);
 737                         if (err) {
 738                                 pr_err("intel_engine_reset(%s) failed, err:%d\n",
 739                                        engine->name, err);
 740                                 break;
 741                         }
 742
 743                         if (i915_reset_count(global) != reset_count) {
 744                                 pr_err("Full GPU reset recorded! (engine reset expected)\n");
 745                                 err = -EINVAL;
 746                                 break;
 747                         }
 748
 749                         if (i915_reset_engine_count(global, engine) !=
 750                             ++reset_engine_count) {
 751                                 pr_err("%s engine reset not recorded!\n",
 752                                        engine->name);
 753                                 err = -EINVAL;
 754                                 break;
 755                         }
 756
 757                         count++;
 758                 } while (time_before(jiffies, end_time));
 759                 clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 760                 st_engine_heartbeat_enable(engine);
 761                 pr_info("%s: Completed %lu %s resets\n",
 762                         engine->name, count, active ? "active" : "idle");
 763
 764                 if (err)
 765                         break;
 766
 767                 err = igt_flush_test(gt->i915);
 768                 if (err)
 769                         break;
 770         }
 771
 772         if (intel_gt_is_wedged(gt))
 773                 err = -EIO;
 774
 775         if (active)
 776                 hang_fini(&h);
 777
 778         return err;
 779 }
 780
 781 static int igt_reset_idle_engine(void *arg)
 782 {
 783         return __igt_reset_engine(arg, false);
 784 }
 785
 786 static int igt_reset_active_engine(void *arg)
 787 {
 788         return __igt_reset_engine(arg, true);
 789 }
 790
 791 struct active_engine {
 792         struct task_struct *task;
 793         struct intel_engine_cs *engine;
 794         unsigned long resets;
 795         unsigned int flags;
 796 };
 797
 798 #define TEST_ACTIVE     BIT(0)
 799 #define TEST_OTHERS     BIT(1)
 800 #define TEST_SELF       BIT(2)
 801 #define TEST_PRIORITY   BIT(3)
 802
 803 static int active_request_put(struct i915_request *rq)
 804 {
 805         int err = 0;
 806
 807         if (!rq)
 808                 return 0;
 809
 810         if (i915_request_wait(rq, 0, 5 * HZ) < 0) {
 811                 GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n",
 812                           rq->engine->name,
 813                           rq->fence.context,
 814                           rq->fence.seqno);
 815                 GEM_TRACE_DUMP();
 816
 817                 intel_gt_set_wedged(rq->engine->gt);
 818                 err = -EIO;
 819         }
 820
 821         i915_request_put(rq);
 822
 823         return err;
 824 }
 825
 826 static int active_engine(void *data)
 827 {
 828         I915_RND_STATE(prng);
 829         struct active_engine *arg = data;
 830         struct intel_engine_cs *engine = arg->engine;
 831         struct i915_request *rq[8] = {};
 832         struct intel_context *ce[ARRAY_SIZE(rq)];
 833         unsigned long count;
 834         int err = 0;
 835
 836         for (count = 0; count < ARRAY_SIZE(ce); count++) {
 837                 ce[count] = intel_context_create(engine);
 838                 if (IS_ERR(ce[count])) {
 839                         err = PTR_ERR(ce[count]);
 840                         while (--count)
 841                                 intel_context_put(ce[count]);
 842                         return err;
 843                 }
 844         }
 845
 846         count = 0;
 847         while (!kthread_should_stop()) {
 848                 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);
 849                 struct i915_request *old = rq[idx];
 850                 struct i915_request *new;
 851
 852                 new = intel_context_create_request(ce[idx]);
 853                 if (IS_ERR(new)) {
 854                         err = PTR_ERR(new);
 855                         break;
 856                 }
 857
 858                 rq[idx] = i915_request_get(new);
 859                 i915_request_add(new);
 860
 861                 if (engine->schedule && arg->flags & TEST_PRIORITY) {
 862                         struct i915_sched_attr attr = {
 863                                 .priority =
 864                                         i915_prandom_u32_max_state(512, &prng),
 865                         };
 866                         engine->schedule(rq[idx], &attr);
 867                 }
 868
 869                 err = active_request_put(old);
 870                 if (err)
 871                         break;
 872
 873                 cond_resched();
 874         }
 875
 876         for (count = 0; count < ARRAY_SIZE(rq); count++) {
 877                 int err__ = active_request_put(rq[count]);
 878
 879                 /* Keep the first error */
 880                 if (!err)
 881                         err = err__;
 882
 883                 intel_context_put(ce[count]);
 884         }
 885
 886         return err;
 887 }
 888
 889 static int __igt_reset_engines(struct intel_gt *gt,
 890                                const char *test_name,
 891                                unsigned int flags)
 892 {
 893         struct i915_gpu_error *global = &gt->i915->gpu_error;
 894         struct intel_engine_cs *engine, *other;
 895         enum intel_engine_id id, tmp;
 896         struct hang h;
 897         int err = 0;
 898
 899         /* Check that issuing a reset on one engine does not interfere
 900          * with any other engine.
 901          */
 902
 903         if (!intel_has_reset_engine(gt))
 904                 return 0;
 905
 906         if (flags & TEST_ACTIVE) {
 907                 err = hang_init(&h, gt);
 908                 if (err)
 909                         return err;
 910
 911                 if (flags & TEST_PRIORITY)
 912                         h.ctx->sched.priority = 1024;
 913         }
 914
 915         for_each_engine(engine, gt, id) {
 916                 struct active_engine threads[I915_NUM_ENGINES] = {};
 917                 unsigned long device = i915_reset_count(global);
 918                 unsigned long count = 0, reported;
 919                 IGT_TIMEOUT(end_time);
 920
 921                 if (flags & TEST_ACTIVE &&
 922                     !intel_engine_can_store_dword(engine))
 923                         continue;
 924
 925                 if (!wait_for_idle(engine)) {
 926                         pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",
 927                                engine->name, test_name);
 928                         err = -EIO;
 929                         break;
 930                 }
 931
 932                 memset(threads, 0, sizeof(threads));
 933                 for_each_engine(other, gt, tmp) {
 934                         struct task_struct *tsk;
 935
 936                         threads[tmp].resets =
 937                                 i915_reset_engine_count(global, other);
 938
 939                         if (other == engine && !(flags & TEST_SELF))
 940                                 continue;
 941
 942                         if (other != engine && !(flags & TEST_OTHERS))
 943                                 continue;
 944
 945                         threads[tmp].engine = other;
 946                         threads[tmp].flags = flags;
 947
 948                         tsk = kthread_run(active_engine, &threads[tmp],
 949                                           "igt/%s", other->name);
 950                         if (IS_ERR(tsk)) {
 951                                 err = PTR_ERR(tsk);
 952                                 goto unwind;
 953                         }
 954
 955                         threads[tmp].task = tsk;
 956                         get_task_struct(tsk);
 957                 }
 958
 959                 yield(); /* start all threads before we begin */
 960
 961                 st_engine_heartbeat_disable(engine);
 962                 set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 963                 do {
 964                         struct i915_request *rq = NULL;
 965
 966                         if (flags & TEST_ACTIVE) {
 967                                 rq = hang_create_request(&h, engine);
 968                                 if (IS_ERR(rq)) {
 969                                         err = PTR_ERR(rq);
 970                                         break;
 971                                 }
 972
 973                                 i915_request_get(rq);
 974                                 i915_request_add(rq);
 975
 976                                 if (!wait_until_running(&h, rq)) {
 977                                         struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
 978
 979                                         pr_err("%s: Failed to start request %llx, at %x\n",
 980                                                __func__, rq->fence.seqno, hws_seqno(&h, rq));
 981                                         intel_engine_dump(engine, &p,
 982                                                           "%s\n", engine->name);
 983
 984                                         i915_request_put(rq);
 985                                         err = -EIO;
 986                                         break;
 987                                 }
 988                         }
 989
 990                         err = intel_engine_reset(engine, NULL);
 991                         if (err) {
 992                                 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",
 993                                        engine->name, test_name, err);
 994                                 break;
 995                         }
 996
 997                         count++;
 998
 999                         if (rq) {
1000                                 if (rq->fence.error != -EIO) {
1001                                         pr_err("i915_reset_engine(%s:%s):"
1002                                                " failed to reset request %llx:%lld\n",
1003                                                engine->name, test_name,
1004                                                rq->fence.context,
1005                                                rq->fence.seqno);
1006                                         i915_request_put(rq);
1007
1008                                         GEM_TRACE_DUMP();
1009                                         intel_gt_set_wedged(gt);
1010                                         err = -EIO;
1011                                         break;
1012                                 }
1013
1014                                 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1015                                         struct drm_printer p =
1016                                                 drm_info_printer(gt->i915->drm.dev);
1017
1018                                         pr_err("i915_reset_engine(%s:%s):"
1019                                                " failed to complete request %llx:%lld after reset\n",
1020                                                engine->name, test_name,
1021                                                rq->fence.context,
1022                                                rq->fence.seqno);
1023                                         intel_engine_dump(engine, &p,
1024                                                           "%s\n", engine->name);
1025                                         i915_request_put(rq);
1026
1027                                         GEM_TRACE_DUMP();
1028                                         intel_gt_set_wedged(gt);
1029                                         err = -EIO;
1030                                         break;
1031                                 }
1032
1033                                 i915_request_put(rq);
1034                         }
1035
1036                         if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {
1037                                 struct drm_printer p =
1038                                         drm_info_printer(gt->i915->drm.dev);
1039
1040                                 pr_err("i915_reset_engine(%s:%s):"
1041                                        " failed to idle after reset\n",
1042                                        engine->name, test_name);
1043                                 intel_engine_dump(engine, &p,
1044                                                   "%s\n", engine->name);
1045
1046                                 err = -EIO;
1047                                 break;
1048                         }
1049                 } while (time_before(jiffies, end_time));
1050                 clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
1051                 st_engine_heartbeat_enable(engine);
1052
1053                 pr_info("i915_reset_engine(%s:%s): %lu resets\n",
1054                         engine->name, test_name, count);
1055
1056                 reported = i915_reset_engine_count(global, engine);
1057                 reported -= threads[engine->id].resets;
1058                 if (reported != count) {
1059                         pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",
1060                                engine->name, test_name, count, reported);
1061                         if (!err)
1062                                 err = -EINVAL;
1063                 }
1064
1065 unwind:
1066                 for_each_engine(other, gt, tmp) {
1067                         int ret;
1068
1069                         if (!threads[tmp].task)
1070                                 continue;
1071
1072                         ret = kthread_stop(threads[tmp].task);
1073                         if (ret) {
1074                                 pr_err("kthread for other engine %s failed, err=%d\n",
1075                                        other->name, ret);
1076                                 if (!err)
1077                                         err = ret;
1078                         }
1079                         put_task_struct(threads[tmp].task);
1080
1081                         if (other->uabi_class != engine->uabi_class &&
1082                             threads[tmp].resets !=
1083                             i915_reset_engine_count(global, other)) {
1084                                 pr_err("Innocent engine %s was reset (count=%ld)\n",
1085                                        other->name,
1086                                        i915_reset_engine_count(global, other) -
1087                                        threads[tmp].resets);
1088                                 if (!err)
1089                                         err = -EINVAL;
1090                         }
1091                 }
1092
1093                 if (device != i915_reset_count(global)) {
1094                         pr_err("Global reset (count=%ld)!\n",
1095                                i915_reset_count(global) - device);
1096                         if (!err)
1097                                 err = -EINVAL;
1098                 }
1099
1100                 if (err)
1101                         break;
1102
1103                 err = igt_flush_test(gt->i915);
1104                 if (err)
1105                         break;
1106         }
1107
1108         if (intel_gt_is_wedged(gt))
1109                 err = -EIO;
1110
1111         if (flags & TEST_ACTIVE)
1112                 hang_fini(&h);
1113
1114         return err;
1115 }
1116
1117 static int igt_reset_engines(void *arg)
1118 {
1119         static const struct {
1120                 const char *name;
1121                 unsigned int flags;
1122         } phases[] = {
1123                 { "idle", 0 },
1124                 { "active", TEST_ACTIVE },
1125                 { "others-idle", TEST_OTHERS },
1126                 { "others-active", TEST_OTHERS | TEST_ACTIVE },
1127                 {
1128                         "others-priority",
1129                         TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY
1130                 },
1131                 {
1132                         "self-priority",
1133                         TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,
1134                 },
1135                 { }
1136         };
1137         struct intel_gt *gt = arg;
1138         typeof(*phases) *p;
1139         int err;
1140
1141         for (p = phases; p->name; p++) {
1142                 if (p->flags & TEST_PRIORITY) {
1143                         if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
1144                                 continue;
1145                 }
1146
1147                 err = __igt_reset_engines(arg, p->name, p->flags);
1148                 if (err)
1149                         return err;
1150         }
1151
1152         return 0;
1153 }
1154
1155 static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask)
1156 {
1157         u32 count = i915_reset_count(&gt->i915->gpu_error);
1158
1159         intel_gt_reset(gt, mask, NULL);
1160
1161         return count;
1162 }
1163
1164 static int igt_reset_wait(void *arg)
1165 {
1166         struct intel_gt *gt = arg;
1167         struct i915_gpu_error *global = &gt->i915->gpu_error;
1168         struct intel_engine_cs *engine = gt->engine[RCS0];
1169         struct i915_request *rq;
1170         unsigned int reset_count;
1171         struct hang h;
1172         long timeout;
1173         int err;
1174
1175         if (!engine || !intel_engine_can_store_dword(engine))
1176                 return 0;
1177
1178         /* Check that we detect a stuck waiter and issue a reset */
1179
1180         igt_global_reset_lock(gt);
1181
1182         err = hang_init(&h, gt);
1183         if (err)
1184                 goto unlock;
1185
1186         rq = hang_create_request(&h, engine);
1187         if (IS_ERR(rq)) {
1188                 err = PTR_ERR(rq);
1189                 goto fini;
1190         }
1191
1192         i915_request_get(rq);
1193         i915_request_add(rq);
1194
1195         if (!wait_until_running(&h, rq)) {
1196                 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1197
1198                 pr_err("%s: Failed to start request %llx, at %x\n",
1199                        __func__, rq->fence.seqno, hws_seqno(&h, rq));
1200                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1201
1202                 intel_gt_set_wedged(gt);
1203
1204                 err = -EIO;
1205                 goto out_rq;
1206         }
1207
1208         reset_count = fake_hangcheck(gt, ALL_ENGINES);
1209
1210         timeout = i915_request_wait(rq, 0, 10);
1211         if (timeout < 0) {
1212                 pr_err("i915_request_wait failed on a stuck request: err=%ld\n",
1213                        timeout);
1214                 err = timeout;
1215                 goto out_rq;
1216         }
1217
1218         if (i915_reset_count(global) == reset_count) {
1219                 pr_err("No GPU reset recorded!\n");
1220                 err = -EINVAL;
1221                 goto out_rq;
1222         }
1223
1224 out_rq:
1225         i915_request_put(rq);
1226 fini:
1227         hang_fini(&h);
1228 unlock:
1229         igt_global_reset_unlock(gt);
1230
1231         if (intel_gt_is_wedged(gt))
1232                 return -EIO;
1233
1234         return err;
1235 }
1236
1237 struct evict_vma {
1238         struct completion completion;
1239         struct i915_vma *vma;
1240 };
1241
1242 static int evict_vma(void *data)
1243 {
1244         struct evict_vma *arg = data;
1245         struct i915_address_space *vm = arg->vma->vm;
1246         struct drm_mm_node evict = arg->vma->node;
1247         int err;
1248
1249         complete(&arg->completion);
1250
1251         mutex_lock(&vm->mutex);
1252         err = i915_gem_evict_for_node(vm, &evict, 0);
1253         mutex_unlock(&vm->mutex);
1254
1255         return err;
1256 }
1257
1258 static int evict_fence(void *data)
1259 {
1260         struct evict_vma *arg = data;
1261         int err;
1262
1263         complete(&arg->completion);
1264
1265         /* Mark the fence register as dirty to force the mmio update. */
1266         err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);
1267         if (err) {
1268                 pr_err("Invalid Y-tiling settings; err:%d\n", err);
1269                 return err;
1270         }
1271
1272         err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE);
1273         if (err) {
1274                 pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err);
1275                 return err;
1276         }
1277
1278         err = i915_vma_pin_fence(arg->vma);
1279         i915_vma_unpin(arg->vma);
1280         if (err) {
1281                 pr_err("Unable to pin Y-tiled fence; err:%d\n", err);
1282                 return err;
1283         }
1284
1285         i915_vma_unpin_fence(arg->vma);
1286
1287         return 0;
1288 }
1289
1290 static int __igt_reset_evict_vma(struct intel_gt *gt,
1291                                  struct i915_address_space *vm,
1292                                  int (*fn)(void *),
1293                                  unsigned int flags)
1294 {
1295         struct intel_engine_cs *engine = gt->engine[RCS0];
1296         struct drm_i915_gem_object *obj;
1297         struct task_struct *tsk = NULL;
1298         struct i915_request *rq;
1299         struct evict_vma arg;
1300         struct hang h;
1301         unsigned int pin_flags;
1302         int err;
1303
1304         if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE)
1305                 return 0;
1306
1307         if (!engine || !intel_engine_can_store_dword(engine))
1308                 return 0;
1309
1310         /* Check that we can recover an unbind stuck on a hanging request */
1311
1312         err = hang_init(&h, gt);
1313         if (err)
1314                 return err;
1315
1316         obj = i915_gem_object_create_internal(gt->i915, SZ_1M);
1317         if (IS_ERR(obj)) {
1318                 err = PTR_ERR(obj);
1319                 goto fini;
1320         }
1321
1322         if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1323                 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);
1324                 if (err) {
1325                         pr_err("Invalid X-tiling settings; err:%d\n", err);
1326                         goto out_obj;
1327                 }
1328         }
1329
1330         arg.vma = i915_vma_instance(obj, vm, NULL);
1331         if (IS_ERR(arg.vma)) {
1332                 err = PTR_ERR(arg.vma);
1333                 goto out_obj;
1334         }
1335
1336         rq = hang_create_request(&h, engine);
1337         if (IS_ERR(rq)) {
1338                 err = PTR_ERR(rq);
1339                 goto out_obj;
1340         }
1341
1342         pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER;
1343
1344         if (flags & EXEC_OBJECT_NEEDS_FENCE)
1345                 pin_flags |= PIN_MAPPABLE;
1346
1347         err = i915_vma_pin(arg.vma, 0, 0, pin_flags);
1348         if (err) {
1349                 i915_request_add(rq);
1350                 goto out_obj;
1351         }
1352
1353         if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1354                 err = i915_vma_pin_fence(arg.vma);
1355                 if (err) {
1356                         pr_err("Unable to pin X-tiled fence; err:%d\n", err);
1357                         i915_vma_unpin(arg.vma);
1358                         i915_request_add(rq);
1359                         goto out_obj;
1360                 }
1361         }
1362
1363         i915_vma_lock(arg.vma);
1364         err = i915_request_await_object(rq, arg.vma->obj,
1365                                         flags & EXEC_OBJECT_WRITE);
1366         if (err == 0)
1367                 err = i915_vma_move_to_active(arg.vma, rq, flags);
1368         i915_vma_unlock(arg.vma);
1369
1370         if (flags & EXEC_OBJECT_NEEDS_FENCE)
1371                 i915_vma_unpin_fence(arg.vma);
1372         i915_vma_unpin(arg.vma);
1373
1374         i915_request_get(rq);
1375         i915_request_add(rq);
1376         if (err)
1377                 goto out_rq;
1378
1379         if (!wait_until_running(&h, rq)) {
1380                 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1381
1382                 pr_err("%s: Failed to start request %llx, at %x\n",
1383                        __func__, rq->fence.seqno, hws_seqno(&h, rq));
1384                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1385
1386                 intel_gt_set_wedged(gt);
1387                 goto out_reset;
1388         }
1389
1390         init_completion(&arg.completion);
1391
1392         tsk = kthread_run(fn, &arg, "igt/evict_vma");
1393         if (IS_ERR(tsk)) {
1394                 err = PTR_ERR(tsk);
1395                 tsk = NULL;
1396                 goto out_reset;
1397         }
1398         get_task_struct(tsk);
1399
1400         wait_for_completion(&arg.completion);
1401
1402         if (wait_for(!list_empty(&rq->fence.cb_list), 10)) {
1403                 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1404
1405                 pr_err("igt/evict_vma kthread did not wait\n");
1406                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1407
1408                 intel_gt_set_wedged(gt);
1409                 goto out_reset;
1410         }
1411
1412 out_reset:
1413         igt_global_reset_lock(gt);
1414         fake_hangcheck(gt, rq->engine->mask);
1415         igt_global_reset_unlock(gt);
1416
1417         if (tsk) {
1418                 struct intel_wedge_me w;
1419
1420                 /* The reset, even indirectly, should take less than 10ms. */
1421                 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
1422                         err = kthread_stop(tsk);
1423
1424                 put_task_struct(tsk);
1425         }
1426
1427 out_rq:
1428         i915_request_put(rq);
1429 out_obj:
1430         i915_gem_object_put(obj);
1431 fini:
1432         hang_fini(&h);
1433         if (intel_gt_is_wedged(gt))
1434                 return -EIO;
1435
1436         return err;
1437 }
1438
1439 static int igt_reset_evict_ggtt(void *arg)
1440 {
1441         struct intel_gt *gt = arg;
1442
1443         return __igt_reset_evict_vma(gt, &gt->ggtt->vm,
1444                                      evict_vma, EXEC_OBJECT_WRITE);
1445 }
1446
1447 static int igt_reset_evict_ppgtt(void *arg)
1448 {
1449         struct intel_gt *gt = arg;
1450         struct i915_ppgtt *ppgtt;
1451         int err;
1452
1453         /* aliasing == global gtt locking, covered above */
1454         if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL)
1455                 return 0;
1456
1457         ppgtt = i915_ppgtt_create(gt);
1458         if (IS_ERR(ppgtt))
1459                 return PTR_ERR(ppgtt);
1460
1461         err = __igt_reset_evict_vma(gt, &ppgtt->vm,
1462                                     evict_vma, EXEC_OBJECT_WRITE);
1463         i915_vm_put(&ppgtt->vm);
1464
1465         return err;
1466 }
1467
1468 static int igt_reset_evict_fence(void *arg)
1469 {
1470         struct intel_gt *gt = arg;
1471
1472         return __igt_reset_evict_vma(gt, &gt->ggtt->vm,
1473                                      evict_fence, EXEC_OBJECT_NEEDS_FENCE);
1474 }
1475
1476 static int wait_for_others(struct intel_gt *gt,
1477                            struct intel_engine_cs *exclude)
1478 {
1479         struct intel_engine_cs *engine;
1480         enum intel_engine_id id;
1481
1482         for_each_engine(engine, gt, id) {
1483                 if (engine == exclude)
1484                         continue;
1485
1486                 if (!wait_for_idle(engine))
1487                         return -EIO;
1488         }
1489
1490         return 0;
1491 }
1492
1493 static int igt_reset_queue(void *arg)
1494 {
1495         struct intel_gt *gt = arg;
1496         struct i915_gpu_error *global = &gt->i915->gpu_error;
1497         struct intel_engine_cs *engine;
1498         enum intel_engine_id id;
1499         struct hang h;
1500         int err;
1501
1502         /* Check that we replay pending requests following a hang */
1503
1504         igt_global_reset_lock(gt);
1505
1506         err = hang_init(&h, gt);
1507         if (err)
1508                 goto unlock;
1509
1510         for_each_engine(engine, gt, id) {
1511                 struct i915_request *prev;
1512                 IGT_TIMEOUT(end_time);
1513                 unsigned int count;
1514
1515                 if (!intel_engine_can_store_dword(engine))
1516                         continue;
1517
1518                 prev = hang_create_request(&h, engine);
1519                 if (IS_ERR(prev)) {
1520                         err = PTR_ERR(prev);
1521                         goto fini;
1522                 }
1523
1524                 i915_request_get(prev);
1525                 i915_request_add(prev);
1526
1527                 count = 0;
1528                 do {
1529                         struct i915_request *rq;
1530                         unsigned int reset_count;
1531
1532                         rq = hang_create_request(&h, engine);
1533                         if (IS_ERR(rq)) {
1534                                 err = PTR_ERR(rq);
1535                                 goto fini;
1536                         }
1537
1538                         i915_request_get(rq);
1539                         i915_request_add(rq);
1540
1541                         /*
1542                          * XXX We don't handle resetting the kernel context
1543                          * very well. If we trigger a device reset twice in
1544                          * quick succession while the kernel context is
1545                          * executing, we may end up skipping the breadcrumb.
1546                          * This is really only a problem for the selftest as
1547                          * normally there is a large interlude between resets
1548                          * (hangcheck), or we focus on resetting just one
1549                          * engine and so avoid repeatedly resetting innocents.
1550                          */
1551                         err = wait_for_others(gt, engine);
1552                         if (err) {
1553                                 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",
1554                                        __func__, engine->name);
1555                                 i915_request_put(rq);
1556                                 i915_request_put(prev);
1557
1558                                 GEM_TRACE_DUMP();
1559                                 intel_gt_set_wedged(gt);
1560                                 goto fini;
1561                         }
1562
1563                         if (!wait_until_running(&h, prev)) {
1564                                 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1565
1566                                 pr_err("%s(%s): Failed to start request %llx, at %x\n",
1567                                        __func__, engine->name,
1568                                        prev->fence.seqno, hws_seqno(&h, prev));
1569                                 intel_engine_dump(engine, &p,
1570                                                   "%s\n", engine->name);
1571
1572                                 i915_request_put(rq);
1573                                 i915_request_put(prev);
1574
1575                                 intel_gt_set_wedged(gt);
1576
1577                                 err = -EIO;
1578                                 goto fini;
1579                         }
1580
1581                         reset_count = fake_hangcheck(gt, BIT(id));
1582
1583                         if (prev->fence.error != -EIO) {
1584                                 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
1585                                        prev->fence.error);
1586                                 i915_request_put(rq);
1587                                 i915_request_put(prev);
1588                                 err = -EINVAL;
1589                                 goto fini;
1590                         }
1591
1592                         if (rq->fence.error) {
1593                                 pr_err("Fence error status not zero [%d] after unrelated reset\n",
1594                                        rq->fence.error);
1595                                 i915_request_put(rq);
1596                                 i915_request_put(prev);
1597                                 err = -EINVAL;
1598                                 goto fini;
1599                         }
1600
1601                         if (i915_reset_count(global) == reset_count) {
1602                                 pr_err("No GPU reset recorded!\n");
1603                                 i915_request_put(rq);
1604                                 i915_request_put(prev);
1605                                 err = -EINVAL;
1606                                 goto fini;
1607                         }
1608
1609                         i915_request_put(prev);
1610                         prev = rq;
1611                         count++;
1612                 } while (time_before(jiffies, end_time));
1613                 pr_info("%s: Completed %d queued resets\n",
1614                         engine->name, count);
1615
1616                 *h.batch = MI_BATCH_BUFFER_END;
1617                 intel_gt_chipset_flush(engine->gt);
1618
1619                 i915_request_put(prev);
1620
1621                 err = igt_flush_test(gt->i915);
1622                 if (err)
1623                         break;
1624         }
1625
1626 fini:
1627         hang_fini(&h);
1628 unlock:
1629         igt_global_reset_unlock(gt);
1630
1631         if (intel_gt_is_wedged(gt))
1632                 return -EIO;
1633
1634         return err;
1635 }
1636
1637 static int igt_handle_error(void *arg)
1638 {
1639         struct intel_gt *gt = arg;
1640         struct i915_gpu_error *global = &gt->i915->gpu_error;
1641         struct intel_engine_cs *engine = gt->engine[RCS0];
1642         struct hang h;
1643         struct i915_request *rq;
1644         struct i915_gpu_coredump *error;
1645         int err;
1646
1647         /* Check that we can issue a global GPU and engine reset */
1648
1649         if (!intel_has_reset_engine(gt))
1650                 return 0;
1651
1652         if (!engine || !intel_engine_can_store_dword(engine))
1653                 return 0;
1654
1655         err = hang_init(&h, gt);
1656         if (err)
1657                 return err;
1658
1659         rq = hang_create_request(&h, engine);
1660         if (IS_ERR(rq)) {
1661                 err = PTR_ERR(rq);
1662                 goto err_fini;
1663         }
1664
1665         i915_request_get(rq);
1666         i915_request_add(rq);
1667
1668         if (!wait_until_running(&h, rq)) {
1669                 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1670
1671                 pr_err("%s: Failed to start request %llx, at %x\n",
1672                        __func__, rq->fence.seqno, hws_seqno(&h, rq));
1673                 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1674
1675                 intel_gt_set_wedged(gt);
1676
1677                 err = -EIO;
1678                 goto err_request;
1679         }
1680
1681         /* Temporarily disable error capture */
1682         error = xchg(&global->first_error, (void *)-1);
1683
1684         intel_gt_handle_error(gt, engine->mask, 0, NULL);
1685
1686         xchg(&global->first_error, error);
1687
1688         if (rq->fence.error != -EIO) {
1689                 pr_err("Guilty request not identified!\n");
1690                 err = -EINVAL;
1691                 goto err_request;
1692         }
1693
1694 err_request:
1695         i915_request_put(rq);
1696 err_fini:
1697         hang_fini(&h);
1698         return err;
1699 }
1700
1701 static int __igt_atomic_reset_engine(struct intel_engine_cs *engine,
1702                                      const struct igt_atomic_section *p,
1703                                      const char *mode)
1704 {
1705         struct tasklet_struct * const t = &engine->execlists.tasklet;
1706         int err;
1707
1708         GEM_TRACE("i915_reset_engine(%s:%s) under %s\n",
1709                   engine->name, mode, p->name);
1710
1711         if (t->func)
1712                 tasklet_disable(t);
1713         if (strcmp(p->name, "softirq"))
1714                 local_bh_disable();
1715         p->critical_section_begin();
1716
1717         err = __intel_engine_reset_bh(engine, NULL);
1718
1719         p->critical_section_end();
1720         if (strcmp(p->name, "softirq"))
1721                 local_bh_enable();
1722         if (t->func) {
1723                 tasklet_enable(t);
1724                 tasklet_hi_schedule(t);
1725         }
1726
1727         if (err)
1728                 pr_err("i915_reset_engine(%s:%s) failed under %s\n",
1729                        engine->name, mode, p->name);
1730
1731         return err;
1732 }
1733
1734 static int igt_atomic_reset_engine(struct intel_engine_cs *engine,
1735                                    const struct igt_atomic_section *p)
1736 {
1737         struct i915_request *rq;
1738         struct hang h;
1739         int err;
1740
1741         err = __igt_atomic_reset_engine(engine, p, "idle");
1742         if (err)
1743                 return err;
1744
1745         err = hang_init(&h, engine->gt);
1746         if (err)
1747                 return err;
1748
1749         rq = hang_create_request(&h, engine);
1750         if (IS_ERR(rq)) {
1751                 err = PTR_ERR(rq);
1752                 goto out;
1753         }
1754
1755         i915_request_get(rq);
1756         i915_request_add(rq);
1757
1758         if (wait_until_running(&h, rq)) {
1759                 err = __igt_atomic_reset_engine(engine, p, "active");
1760         } else {
1761                 pr_err("%s(%s): Failed to start request %llx, at %x\n",
1762                        __func__, engine->name,
1763                        rq->fence.seqno, hws_seqno(&h, rq));
1764                 intel_gt_set_wedged(engine->gt);
1765                 err = -EIO;
1766         }
1767
1768         if (err == 0) {
1769                 struct intel_wedge_me w;
1770
1771                 intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */)
1772                         i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1773                 if (intel_gt_is_wedged(engine->gt))
1774                         err = -EIO;
1775         }
1776
1777         i915_request_put(rq);
1778 out:
1779         hang_fini(&h);
1780         return err;
1781 }
1782
1783 static int igt_reset_engines_atomic(void *arg)
1784 {
1785         struct intel_gt *gt = arg;
1786         const typeof(*igt_atomic_phases) *p;
1787         int err = 0;
1788
1789         /* Check that the engines resets are usable from atomic context */
1790
1791         if (!intel_has_reset_engine(gt))
1792                 return 0;
1793
1794         if (intel_uc_uses_guc_submission(&gt->uc))
1795                 return 0;
1796
1797         igt_global_reset_lock(gt);
1798
1799         /* Flush any requests before we get started and check basics */
1800         if (!igt_force_reset(gt))
1801                 goto unlock;
1802
1803         for (p = igt_atomic_phases; p->name; p++) {
1804                 struct intel_engine_cs *engine;
1805                 enum intel_engine_id id;
1806
1807                 for_each_engine(engine, gt, id) {
1808                         err = igt_atomic_reset_engine(engine, p);
1809                         if (err)
1810                                 goto out;
1811                 }
1812         }
1813
1814 out:
1815         /* As we poke around the guts, do a full reset before continuing. */
1816         igt_force_reset(gt);
1817 unlock:
1818         igt_global_reset_unlock(gt);
1819
1820         return err;
1821 }
1822
1823 int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
1824 {
1825         static const struct i915_subtest tests[] = {
1826                 SUBTEST(igt_hang_sanitycheck),
1827                 SUBTEST(igt_reset_nop),
1828                 SUBTEST(igt_reset_nop_engine),
1829                 SUBTEST(igt_reset_idle_engine),
1830                 SUBTEST(igt_reset_active_engine),
1831                 SUBTEST(igt_reset_fail_engine),
1832                 SUBTEST(igt_reset_engines),
1833                 SUBTEST(igt_reset_engines_atomic),
1834                 SUBTEST(igt_reset_queue),
1835                 SUBTEST(igt_reset_wait),
1836                 SUBTEST(igt_reset_evict_ggtt),
1837                 SUBTEST(igt_reset_evict_ppgtt),
1838                 SUBTEST(igt_reset_evict_fence),
1839                 SUBTEST(igt_handle_error),
1840         };
1841         struct intel_gt *gt = &i915->gt;
1842         intel_wakeref_t wakeref;
1843         int err;
1844
1845         if (!intel_has_gpu_reset(gt))
1846                 return 0;
1847
1848         if (intel_gt_is_wedged(gt))
1849                 return -EIO; /* we're long past hope of a successful reset */
1850
1851         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1852
1853         err = intel_gt_live_subtests(tests, gt);
1854
1855         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1856
1857         return err;
1858 }