drivers/gpu/drm/i915/gt/intel_ring_submission.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2008-2021 Intel Corporation
   4  */
   5
   6 #include "gen2_engine_cs.h"
   7 #include "gen6_engine_cs.h"
   8 #include "gen6_ppgtt.h"
   9 #include "gen7_renderclear.h"
  10 #include "i915_drv.h"
  11 #include "i915_mitigations.h"
  12 #include "intel_breadcrumbs.h"
  13 #include "intel_context.h"
  14 #include "intel_gt.h"
  15 #include "intel_gt_irq.h"
  16 #include "intel_reset.h"
  17 #include "intel_ring.h"
  18 #include "shmem_utils.h"
  19
  20 /* Rough estimate of the typical request size, performing a flush,
  21  * set-context and then emitting the batch.
  22  */
  23 #define LEGACY_REQUEST_SIZE 200
  24
  25 static void set_hwstam(struct intel_engine_cs *engine, u32 mask)
  26 {
  27         /*
  28          * Keep the render interrupt unmasked as this papers over
  29          * lost interrupts following a reset.
  30          */
  31         if (engine->class == RENDER_CLASS) {
  32                 if (GRAPHICS_VER(engine->i915) >= 6)
  33                         mask &= ~BIT(0);
  34                 else
  35                         mask &= ~I915_USER_INTERRUPT;
  36         }
  37
  38         intel_engine_set_hwsp_writemask(engine, mask);
  39 }
  40
  41 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys)
  42 {
  43         u32 addr;
  44
  45         addr = lower_32_bits(phys);
  46         if (GRAPHICS_VER(engine->i915) >= 4)
  47                 addr |= (phys >> 28) & 0xf0;
  48
  49         intel_uncore_write(engine->uncore, HWS_PGA, addr);
  50 }
  51
  52 static struct page *status_page(struct intel_engine_cs *engine)
  53 {
  54         struct drm_i915_gem_object *obj = engine->status_page.vma->obj;
  55
  56         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
  57         return sg_page(obj->mm.pages->sgl);
  58 }
  59
  60 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
  61 {
  62         set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine))));
  63         set_hwstam(engine, ~0u);
  64 }
  65
  66 static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
  67 {
  68         i915_reg_t hwsp;
  69
  70         /*
  71          * The ring status page addresses are no longer next to the rest of
  72          * the ring registers as of gen7.
  73          */
  74         if (GRAPHICS_VER(engine->i915) == 7) {
  75                 switch (engine->id) {
  76                 /*
  77                  * No more rings exist on Gen7. Default case is only to shut up
  78                  * gcc switch check warning.
  79                  */
  80                 default:
  81                         GEM_BUG_ON(engine->id);
  82                         fallthrough;
  83                 case RCS0:
  84                         hwsp = RENDER_HWS_PGA_GEN7;
  85                         break;
  86                 case BCS0:
  87                         hwsp = BLT_HWS_PGA_GEN7;
  88                         break;
  89                 case VCS0:
  90                         hwsp = BSD_HWS_PGA_GEN7;
  91                         break;
  92                 case VECS0:
  93                         hwsp = VEBOX_HWS_PGA_GEN7;
  94                         break;
  95                 }
  96         } else if (GRAPHICS_VER(engine->i915) == 6) {
  97                 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base);
  98         } else {
  99                 hwsp = RING_HWS_PGA(engine->mmio_base);
 100         }
 101
 102         intel_uncore_write_fw(engine->uncore, hwsp, offset);
 103         intel_uncore_posting_read_fw(engine->uncore, hwsp);
 104 }
 105
 106 static void flush_cs_tlb(struct intel_engine_cs *engine)
 107 {
 108         if (!IS_GRAPHICS_VER(engine->i915, 6, 7))
 109                 return;
 110
 111         /* ring should be idle before issuing a sync flush*/
 112         GEM_DEBUG_WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
 113
 114         ENGINE_WRITE_FW(engine, RING_INSTPM,
 115                         _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
 116                                            INSTPM_SYNC_FLUSH));
 117         if (__intel_wait_for_register_fw(engine->uncore,
 118                                          RING_INSTPM(engine->mmio_base),
 119                                          INSTPM_SYNC_FLUSH, 0,
 120                                          2000, 0, NULL))
 121                 ENGINE_TRACE(engine,
 122                              "wait for SyncFlush to complete for TLB invalidation timed out\n");
 123 }
 124
 125 static void ring_setup_status_page(struct intel_engine_cs *engine)
 126 {
 127         set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma));
 128         set_hwstam(engine, ~0u);
 129
 130         flush_cs_tlb(engine);
 131 }
 132
 133 static struct i915_address_space *vm_alias(struct i915_address_space *vm)
 134 {
 135         if (i915_is_ggtt(vm))
 136                 vm = &i915_vm_to_ggtt(vm)->alias->vm;
 137
 138         return vm;
 139 }
 140
 141 static u32 pp_dir(struct i915_address_space *vm)
 142 {
 143         return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir;
 144 }
 145
 146 static void set_pp_dir(struct intel_engine_cs *engine)
 147 {
 148         struct i915_address_space *vm = vm_alias(engine->gt->vm);
 149
 150         if (!vm)
 151                 return;
 152
 153         ENGINE_WRITE_FW(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G);
 154         ENGINE_WRITE_FW(engine, RING_PP_DIR_BASE, pp_dir(vm));
 155
 156         if (GRAPHICS_VER(engine->i915) >= 7) {
 157                 ENGINE_WRITE_FW(engine,
 158                                 RING_MODE_GEN7,
 159                                 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 160         }
 161 }
 162
 163 static bool stop_ring(struct intel_engine_cs *engine)
 164 {
 165         /* Empty the ring by skipping to the end */
 166         ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL));
 167         ENGINE_POSTING_READ(engine, RING_HEAD);
 168
 169         /* The ring must be empty before it is disabled */
 170         ENGINE_WRITE_FW(engine, RING_CTL, 0);
 171         ENGINE_POSTING_READ(engine, RING_CTL);
 172
 173         /* Then reset the disabled ring */
 174         ENGINE_WRITE_FW(engine, RING_HEAD, 0);
 175         ENGINE_WRITE_FW(engine, RING_TAIL, 0);
 176
 177         return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0;
 178 }
 179
 180 static int xcs_resume(struct intel_engine_cs *engine)
 181 {
 182         struct intel_ring *ring = engine->legacy.ring;
 183
 184         ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",
 185                      ring->head, ring->tail);
 186
 187         /*
 188          * Double check the ring is empty & disabled before we resume. Called
 189          * from atomic context during PCI probe, so _hardirq().
 190          */
 191         intel_synchronize_hardirq(engine->i915);
 192         if (!stop_ring(engine))
 193                 goto err;
 194
 195         if (HWS_NEEDS_PHYSICAL(engine->i915))
 196                 ring_setup_phys_status_page(engine);
 197         else
 198                 ring_setup_status_page(engine);
 199
 200         intel_breadcrumbs_reset(engine->breadcrumbs);
 201
 202         /* Enforce ordering by reading HEAD register back */
 203         ENGINE_POSTING_READ(engine, RING_HEAD);
 204
 205         /*
 206          * Initialize the ring. This must happen _after_ we've cleared the ring
 207          * registers with the above sequence (the readback of the HEAD registers
 208          * also enforces ordering), otherwise the hw might lose the new ring
 209          * register values.
 210          */
 211         ENGINE_WRITE_FW(engine, RING_START, i915_ggtt_offset(ring->vma));
 212
 213         /* Check that the ring offsets point within the ring! */
 214         GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
 215         GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
 216         intel_ring_update_space(ring);
 217
 218         set_pp_dir(engine);
 219
 220         /* First wake the ring up to an empty/idle ring */
 221         ENGINE_WRITE_FW(engine, RING_HEAD, ring->head);
 222         ENGINE_WRITE_FW(engine, RING_TAIL, ring->head);
 223         ENGINE_POSTING_READ(engine, RING_TAIL);
 224
 225         ENGINE_WRITE_FW(engine, RING_CTL,
 226                         RING_CTL_SIZE(ring->size) | RING_VALID);
 227
 228         /* If the head is still not zero, the ring is dead */
 229         if (__intel_wait_for_register_fw(engine->uncore,
 230                                          RING_CTL(engine->mmio_base),
 231                                          RING_VALID, RING_VALID,
 232                                          5000, 0, NULL))
 233                 goto err;
 234
 235         if (GRAPHICS_VER(engine->i915) > 2)
 236                 ENGINE_WRITE_FW(engine,
 237                                 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
 238
 239         /* Now awake, let it get started */
 240         if (ring->tail != ring->head) {
 241                 ENGINE_WRITE_FW(engine, RING_TAIL, ring->tail);
 242                 ENGINE_POSTING_READ(engine, RING_TAIL);
 243         }
 244
 245         /* Papering over lost _interrupts_ immediately following the restart */
 246         intel_engine_signal_breadcrumbs(engine);
 247         return 0;
 248
 249 err:
 250         drm_err(&engine->i915->drm,
 251                 "%s initialization failed; "
 252                 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
 253                 engine->name,
 254                 ENGINE_READ(engine, RING_CTL),
 255                 ENGINE_READ(engine, RING_CTL) & RING_VALID,
 256                 ENGINE_READ(engine, RING_HEAD), ring->head,
 257                 ENGINE_READ(engine, RING_TAIL), ring->tail,
 258                 ENGINE_READ(engine, RING_START),
 259                 i915_ggtt_offset(ring->vma));
 260         return -EIO;
 261 }
 262
 263 static void sanitize_hwsp(struct intel_engine_cs *engine)
 264 {
 265         struct intel_timeline *tl;
 266
 267         list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
 268                 intel_timeline_reset_seqno(tl);
 269 }
 270
 271 static void xcs_sanitize(struct intel_engine_cs *engine)
 272 {
 273         /*
 274          * Poison residual state on resume, in case the suspend didn't!
 275          *
 276          * We have to assume that across suspend/resume (or other loss
 277          * of control) that the contents of our pinned buffers has been
 278          * lost, replaced by garbage. Since this doesn't always happen,
 279          * let's poison such state so that we more quickly spot when
 280          * we falsely assume it has been preserved.
 281          */
 282         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 283                 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
 284
 285         /*
 286          * The kernel_context HWSP is stored in the status_page. As above,
 287          * that may be lost on resume/initialisation, and so we need to
 288          * reset the value in the HWSP.
 289          */
 290         sanitize_hwsp(engine);
 291
 292         /* And scrub the dirty cachelines for the HWSP */
 293         clflush_cache_range(engine->status_page.addr, PAGE_SIZE);
 294 }
 295
 296 static void reset_prepare(struct intel_engine_cs *engine)
 297 {
 298         /*
 299          * We stop engines, otherwise we might get failed reset and a
 300          * dead gpu (on elk). Also as modern gpu as kbl can suffer
 301          * from system hang if batchbuffer is progressing when
 302          * the reset is issued, regardless of READY_TO_RESET ack.
 303          * Thus assume it is best to stop engines on all gens
 304          * where we have a gpu reset.
 305          *
 306          * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
 307          *
 308          * WaMediaResetMainRingCleanup:ctg,elk (presumably)
 309          * WaClearRingBufHeadRegAtInit:ctg,elk
 310          *
 311          * FIXME: Wa for more modern gens needs to be validated
 312          */
 313         ENGINE_TRACE(engine, "\n");
 314         intel_engine_stop_cs(engine);
 315
 316         if (!stop_ring(engine)) {
 317                 /* G45 ring initialization often fails to reset head to zero */
 318                 ENGINE_TRACE(engine,
 319                              "HEAD not reset to zero, "
 320                              "{ CTL:%08x, HEAD:%08x, TAIL:%08x, START:%08x }\n",
 321                              ENGINE_READ_FW(engine, RING_CTL),
 322                              ENGINE_READ_FW(engine, RING_HEAD),
 323                              ENGINE_READ_FW(engine, RING_TAIL),
 324                              ENGINE_READ_FW(engine, RING_START));
 325                 if (!stop_ring(engine)) {
 326                         drm_err(&engine->i915->drm,
 327                                 "failed to set %s head to zero "
 328                                 "ctl %08x head %08x tail %08x start %08x\n",
 329                                 engine->name,
 330                                 ENGINE_READ_FW(engine, RING_CTL),
 331                                 ENGINE_READ_FW(engine, RING_HEAD),
 332                                 ENGINE_READ_FW(engine, RING_TAIL),
 333                                 ENGINE_READ_FW(engine, RING_START));
 334                 }
 335         }
 336 }
 337
 338 static void reset_rewind(struct intel_engine_cs *engine, bool stalled)
 339 {
 340         struct i915_request *pos, *rq;
 341         unsigned long flags;
 342         u32 head;
 343
 344         rq = NULL;
 345         spin_lock_irqsave(&engine->sched_engine->lock, flags);
 346         rcu_read_lock();
 347         list_for_each_entry(pos, &engine->sched_engine->requests, sched.link) {
 348                 if (!__i915_request_is_complete(pos)) {
 349                         rq = pos;
 350                         break;
 351                 }
 352         }
 353         rcu_read_unlock();
 354
 355         /*
 356          * The guilty request will get skipped on a hung engine.
 357          *
 358          * Users of client default contexts do not rely on logical
 359          * state preserved between batches so it is safe to execute
 360          * queued requests following the hang. Non default contexts
 361          * rely on preserved state, so skipping a batch loses the
 362          * evolution of the state and it needs to be considered corrupted.
 363          * Executing more queued batches on top of corrupted state is
 364          * risky. But we take the risk by trying to advance through
 365          * the queued requests in order to make the client behaviour
 366          * more predictable around resets, by not throwing away random
 367          * amount of batches it has prepared for execution. Sophisticated
 368          * clients can use gem_reset_stats_ioctl and dma fence status
 369          * (exported via sync_file info ioctl on explicit fences) to observe
 370          * when it loses the context state and should rebuild accordingly.
 371          *
 372          * The context ban, and ultimately the client ban, mechanism are safety
 373          * valves if client submission ends up resulting in nothing more than
 374          * subsequent hangs.
 375          */
 376
 377         if (rq) {
 378                 /*
 379                  * Try to restore the logical GPU state to match the
 380                  * continuation of the request queue. If we skip the
 381                  * context/PD restore, then the next request may try to execute
 382                  * assuming that its context is valid and loaded on the GPU and
 383                  * so may try to access invalid memory, prompting repeated GPU
 384                  * hangs.
 385                  *
 386                  * If the request was guilty, we still restore the logical
 387                  * state in case the next request requires it (e.g. the
 388                  * aliasing ppgtt), but skip over the hung batch.
 389                  *
 390                  * If the request was innocent, we try to replay the request
 391                  * with the restored context.
 392                  */
 393                 __i915_request_reset(rq, stalled);
 394
 395                 GEM_BUG_ON(rq->ring != engine->legacy.ring);
 396                 head = rq->head;
 397         } else {
 398                 head = engine->legacy.ring->tail;
 399         }
 400         engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head);
 401
 402         spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 403 }
 404
 405 static void reset_finish(struct intel_engine_cs *engine)
 406 {
 407 }
 408
 409 static void reset_cancel(struct intel_engine_cs *engine)
 410 {
 411         struct i915_request *request;
 412         unsigned long flags;
 413
 414         spin_lock_irqsave(&engine->sched_engine->lock, flags);
 415
 416         /* Mark all submitted requests as skipped. */
 417         list_for_each_entry(request, &engine->sched_engine->requests, sched.link)
 418                 i915_request_put(i915_request_mark_eio(request));
 419         intel_engine_signal_breadcrumbs(engine);
 420
 421         /* Remaining _unready_ requests will be nop'ed when submitted */
 422
 423         spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 424 }
 425
 426 static void i9xx_submit_request(struct i915_request *request)
 427 {
 428         i915_request_submit(request);
 429         wmb(); /* paranoid flush writes out of the WCB before mmio */
 430
 431         ENGINE_WRITE(request->engine, RING_TAIL,
 432                      intel_ring_set_tail(request->ring, request->tail));
 433 }
 434
 435 static void __ring_context_fini(struct intel_context *ce)
 436 {
 437         i915_vma_put(ce->state);
 438 }
 439
 440 static void ring_context_destroy(struct kref *ref)
 441 {
 442         struct intel_context *ce = container_of(ref, typeof(*ce), ref);
 443
 444         GEM_BUG_ON(intel_context_is_pinned(ce));
 445
 446         if (ce->state)
 447                 __ring_context_fini(ce);
 448
 449         intel_context_fini(ce);
 450         intel_context_free(ce);
 451 }
 452
 453 static int ring_context_init_default_state(struct intel_context *ce,
 454                                            struct i915_gem_ww_ctx *ww)
 455 {
 456         struct drm_i915_gem_object *obj = ce->state->obj;
 457         void *vaddr;
 458
 459         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 460         if (IS_ERR(vaddr))
 461                 return PTR_ERR(vaddr);
 462
 463         shmem_read(ce->engine->default_state, 0,
 464                    vaddr, ce->engine->context_size);
 465
 466         i915_gem_object_flush_map(obj);
 467         __i915_gem_object_release_map(obj);
 468
 469         __set_bit(CONTEXT_VALID_BIT, &ce->flags);
 470         return 0;
 471 }
 472
 473 static int ring_context_pre_pin(struct intel_context *ce,
 474                                 struct i915_gem_ww_ctx *ww,
 475                                 void **unused)
 476 {
 477         struct i915_address_space *vm;
 478         int err = 0;
 479
 480         if (ce->engine->default_state &&
 481             !test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
 482                 err = ring_context_init_default_state(ce, ww);
 483                 if (err)
 484                         return err;
 485         }
 486
 487         vm = vm_alias(ce->vm);
 488         if (vm)
 489                 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);
 490
 491         return err;
 492 }
 493
 494 static void __context_unpin_ppgtt(struct intel_context *ce)
 495 {
 496         struct i915_address_space *vm;
 497
 498         vm = vm_alias(ce->vm);
 499         if (vm)
 500                 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm));
 501 }
 502
 503 static void ring_context_unpin(struct intel_context *ce)
 504 {
 505 }
 506
 507 static void ring_context_post_unpin(struct intel_context *ce)
 508 {
 509         __context_unpin_ppgtt(ce);
 510 }
 511
 512 static struct i915_vma *
 513 alloc_context_vma(struct intel_engine_cs *engine)
 514 {
 515         struct drm_i915_private *i915 = engine->i915;
 516         struct drm_i915_gem_object *obj;
 517         struct i915_vma *vma;
 518         int err;
 519
 520         obj = i915_gem_object_create_shmem(i915, engine->context_size);
 521         if (IS_ERR(obj))
 522                 return ERR_CAST(obj);
 523
 524         /*
 525          * Try to make the context utilize L3 as well as LLC.
 526          *
 527          * On VLV we don't have L3 controls in the PTEs so we
 528          * shouldn't touch the cache level, especially as that
 529          * would make the object snooped which might have a
 530          * negative performance impact.
 531          *
 532          * Snooping is required on non-llc platforms in execlist
 533          * mode, but since all GGTT accesses use PAT entry 0 we
 534          * get snooping anyway regardless of cache_level.
 535          *
 536          * This is only applicable for Ivy Bridge devices since
 537          * later platforms don't have L3 control bits in the PTE.
 538          */
 539         if (IS_IVYBRIDGE(i915))
 540                 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
 541
 542         vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
 543         if (IS_ERR(vma)) {
 544                 err = PTR_ERR(vma);
 545                 goto err_obj;
 546         }
 547
 548         return vma;
 549
 550 err_obj:
 551         i915_gem_object_put(obj);
 552         return ERR_PTR(err);
 553 }
 554
 555 static int ring_context_alloc(struct intel_context *ce)
 556 {
 557         struct intel_engine_cs *engine = ce->engine;
 558
 559         /* One ringbuffer to rule them all */
 560         GEM_BUG_ON(!engine->legacy.ring);
 561         ce->ring = engine->legacy.ring;
 562         ce->timeline = intel_timeline_get(engine->legacy.timeline);
 563
 564         GEM_BUG_ON(ce->state);
 565         if (engine->context_size) {
 566                 struct i915_vma *vma;
 567
 568                 vma = alloc_context_vma(engine);
 569                 if (IS_ERR(vma))
 570                         return PTR_ERR(vma);
 571
 572                 ce->state = vma;
 573         }
 574
 575         return 0;
 576 }
 577
 578 static int ring_context_pin(struct intel_context *ce, void *unused)
 579 {
 580         return 0;
 581 }
 582
 583 static void ring_context_reset(struct intel_context *ce)
 584 {
 585         intel_ring_reset(ce->ring, ce->ring->emit);
 586         clear_bit(CONTEXT_VALID_BIT, &ce->flags);
 587 }
 588
 589 static const struct intel_context_ops ring_context_ops = {
 590         .alloc = ring_context_alloc,
 591
 592         .pre_pin = ring_context_pre_pin,
 593         .pin = ring_context_pin,
 594         .unpin = ring_context_unpin,
 595         .post_unpin = ring_context_post_unpin,
 596
 597         .enter = intel_context_enter_engine,
 598         .exit = intel_context_exit_engine,
 599
 600         .reset = ring_context_reset,
 601         .destroy = ring_context_destroy,
 602 };
 603
 604 static int load_pd_dir(struct i915_request *rq,
 605                        struct i915_address_space *vm,
 606                        u32 valid)
 607 {
 608         const struct intel_engine_cs * const engine = rq->engine;
 609         u32 *cs;
 610
 611         cs = intel_ring_begin(rq, 12);
 612         if (IS_ERR(cs))
 613                 return PTR_ERR(cs);
 614
 615         *cs++ = MI_LOAD_REGISTER_IMM(1);
 616         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
 617         *cs++ = valid;
 618
 619         *cs++ = MI_LOAD_REGISTER_IMM(1);
 620         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
 621         *cs++ = pp_dir(vm);
 622
 623         /* Stall until the page table load is complete? */
 624         *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 625         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
 626         *cs++ = intel_gt_scratch_offset(engine->gt,
 627                                         INTEL_GT_SCRATCH_FIELD_DEFAULT);
 628
 629         *cs++ = MI_LOAD_REGISTER_IMM(1);
 630         *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
 631         *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);
 632
 633         intel_ring_advance(rq, cs);
 634
 635         return rq->engine->emit_flush(rq, EMIT_FLUSH);
 636 }
 637
 638 static int mi_set_context(struct i915_request *rq,
 639                           struct intel_context *ce,
 640                           u32 flags)
 641 {
 642         struct intel_engine_cs *engine = rq->engine;
 643         struct drm_i915_private *i915 = engine->i915;
 644         enum intel_engine_id id;
 645         const int num_engines =
 646                 IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0;
 647         bool force_restore = false;
 648         int len;
 649         u32 *cs;
 650
 651         len = 4;
 652         if (GRAPHICS_VER(i915) == 7)
 653                 len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
 654         else if (GRAPHICS_VER(i915) == 5)
 655                 len += 2;
 656         if (flags & MI_FORCE_RESTORE) {
 657                 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT);
 658                 flags &= ~MI_FORCE_RESTORE;
 659                 force_restore = true;
 660                 len += 2;
 661         }
 662
 663         cs = intel_ring_begin(rq, len);
 664         if (IS_ERR(cs))
 665                 return PTR_ERR(cs);
 666
 667         /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
 668         if (GRAPHICS_VER(i915) == 7) {
 669                 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
 670                 if (num_engines) {
 671                         struct intel_engine_cs *signaller;
 672
 673                         *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
 674                         for_each_engine(signaller, engine->gt, id) {
 675                                 if (signaller == engine)
 676                                         continue;
 677
 678                                 *cs++ = i915_mmio_reg_offset(
 679                                            RING_PSMI_CTL(signaller->mmio_base));
 680                                 *cs++ = _MASKED_BIT_ENABLE(
 681                                                 GEN6_PSMI_SLEEP_MSG_DISABLE);
 682                         }
 683                 }
 684         } else if (GRAPHICS_VER(i915) == 5) {
 685                 /*
 686                  * This w/a is only listed for pre-production ilk a/b steppings,
 687                  * but is also mentioned for programming the powerctx. To be
 688                  * safe, just apply the workaround; we do not use SyncFlush so
 689                  * this should never take effect and so be a no-op!
 690                  */
 691                 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;
 692         }
 693
 694         if (force_restore) {
 695                 /*
 696                  * The HW doesn't handle being told to restore the current
 697                  * context very well. Quite often it likes goes to go off and
 698                  * sulk, especially when it is meant to be reloading PP_DIR.
 699                  * A very simple fix to force the reload is to simply switch
 700                  * away from the current context and back again.
 701                  *
 702                  * Note that the kernel_context will contain random state
 703                  * following the INHIBIT_RESTORE. We accept this since we
 704                  * never use the kernel_context state; it is merely a
 705                  * placeholder we use to flush other contexts.
 706                  */
 707                 *cs++ = MI_SET_CONTEXT;
 708                 *cs++ = i915_ggtt_offset(engine->kernel_context->state) |
 709                         MI_MM_SPACE_GTT |
 710                         MI_RESTORE_INHIBIT;
 711         }
 712
 713         *cs++ = MI_NOOP;
 714         *cs++ = MI_SET_CONTEXT;
 715         *cs++ = i915_ggtt_offset(ce->state) | flags;
 716         /*
 717          * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
 718          * WaMiSetContext_Hang:snb,ivb,vlv
 719          */
 720         *cs++ = MI_NOOP;
 721
 722         if (GRAPHICS_VER(i915) == 7) {
 723                 if (num_engines) {
 724                         struct intel_engine_cs *signaller;
 725                         i915_reg_t last_reg = {}; /* keep gcc quiet */
 726
 727                         *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
 728                         for_each_engine(signaller, engine->gt, id) {
 729                                 if (signaller == engine)
 730                                         continue;
 731
 732                                 last_reg = RING_PSMI_CTL(signaller->mmio_base);
 733                                 *cs++ = i915_mmio_reg_offset(last_reg);
 734                                 *cs++ = _MASKED_BIT_DISABLE(
 735                                                 GEN6_PSMI_SLEEP_MSG_DISABLE);
 736                         }
 737
 738                         /* Insert a delay before the next switch! */
 739                         *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 740                         *cs++ = i915_mmio_reg_offset(last_reg);
 741                         *cs++ = intel_gt_scratch_offset(engine->gt,
 742                                                         INTEL_GT_SCRATCH_FIELD_DEFAULT);
 743                         *cs++ = MI_NOOP;
 744                 }
 745                 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 746         } else if (GRAPHICS_VER(i915) == 5) {
 747                 *cs++ = MI_SUSPEND_FLUSH;
 748         }
 749
 750         intel_ring_advance(rq, cs);
 751
 752         return 0;
 753 }
 754
 755 static int remap_l3_slice(struct i915_request *rq, int slice)
 756 {
 757 #define L3LOG_DW (GEN7_L3LOG_SIZE / sizeof(u32))
 758         u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice];
 759         int i;
 760
 761         if (!remap_info)
 762                 return 0;
 763
 764         cs = intel_ring_begin(rq, L3LOG_DW * 2 + 2);
 765         if (IS_ERR(cs))
 766                 return PTR_ERR(cs);
 767
 768         /*
 769          * Note: We do not worry about the concurrent register cacheline hang
 770          * here because no other code should access these registers other than
 771          * at initialization time.
 772          */
 773         *cs++ = MI_LOAD_REGISTER_IMM(L3LOG_DW);
 774         for (i = 0; i < L3LOG_DW; i++) {
 775                 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
 776                 *cs++ = remap_info[i];
 777         }
 778         *cs++ = MI_NOOP;
 779         intel_ring_advance(rq, cs);
 780
 781         return 0;
 782 #undef L3LOG_DW
 783 }
 784
 785 static int remap_l3(struct i915_request *rq)
 786 {
 787         struct i915_gem_context *ctx = i915_request_gem_context(rq);
 788         int i, err;
 789
 790         if (!ctx || !ctx->remap_slice)
 791                 return 0;
 792
 793         for (i = 0; i < MAX_L3_SLICES; i++) {
 794                 if (!(ctx->remap_slice & BIT(i)))
 795                         continue;
 796
 797                 err = remap_l3_slice(rq, i);
 798                 if (err)
 799                         return err;
 800         }
 801
 802         ctx->remap_slice = 0;
 803         return 0;
 804 }
 805
 806 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm)
 807 {
 808         int ret;
 809
 810         if (!vm)
 811                 return 0;
 812
 813         ret = rq->engine->emit_flush(rq, EMIT_FLUSH);
 814         if (ret)
 815                 return ret;
 816
 817         /*
 818          * Not only do we need a full barrier (post-sync write) after
 819          * invalidating the TLBs, but we need to wait a little bit
 820          * longer. Whether this is merely delaying us, or the
 821          * subsequent flush is a key part of serialising with the
 822          * post-sync op, this extra pass appears vital before a
 823          * mm switch!
 824          */
 825         ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G);
 826         if (ret)
 827                 return ret;
 828
 829         return rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 830 }
 831
 832 static int clear_residuals(struct i915_request *rq)
 833 {
 834         struct intel_engine_cs *engine = rq->engine;
 835         int ret;
 836
 837         ret = switch_mm(rq, vm_alias(engine->kernel_context->vm));
 838         if (ret)
 839                 return ret;
 840
 841         if (engine->kernel_context->state) {
 842                 ret = mi_set_context(rq,
 843                                      engine->kernel_context,
 844                                      MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT);
 845                 if (ret)
 846                         return ret;
 847         }
 848
 849         ret = engine->emit_bb_start(rq,
 850                                     engine->wa_ctx.vma->node.start, 0,
 851                                     0);
 852         if (ret)
 853                 return ret;
 854
 855         ret = engine->emit_flush(rq, EMIT_FLUSH);
 856         if (ret)
 857                 return ret;
 858
 859         /* Always invalidate before the next switch_mm() */
 860         return engine->emit_flush(rq, EMIT_INVALIDATE);
 861 }
 862
 863 static int switch_context(struct i915_request *rq)
 864 {
 865         struct intel_engine_cs *engine = rq->engine;
 866         struct intel_context *ce = rq->context;
 867         void **residuals = NULL;
 868         int ret;
 869
 870         GEM_BUG_ON(HAS_EXECLISTS(engine->i915));
 871
 872         if (engine->wa_ctx.vma && ce != engine->kernel_context) {
 873                 if (engine->wa_ctx.vma->private != ce &&
 874                     i915_mitigate_clear_residuals()) {
 875                         ret = clear_residuals(rq);
 876                         if (ret)
 877                                 return ret;
 878
 879                         residuals = &engine->wa_ctx.vma->private;
 880                 }
 881         }
 882
 883         ret = switch_mm(rq, vm_alias(ce->vm));
 884         if (ret)
 885                 return ret;
 886
 887         if (ce->state) {
 888                 u32 flags;
 889
 890                 GEM_BUG_ON(engine->id != RCS0);
 891
 892                 /* For resource streamer on HSW+ and power context elsewhere */
 893                 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN);
 894                 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN);
 895
 896                 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT;
 897                 if (test_bit(CONTEXT_VALID_BIT, &ce->flags))
 898                         flags |= MI_RESTORE_EXT_STATE_EN;
 899                 else
 900                         flags |= MI_RESTORE_INHIBIT;
 901
 902                 ret = mi_set_context(rq, ce, flags);
 903                 if (ret)
 904                         return ret;
 905         }
 906
 907         ret = remap_l3(rq);
 908         if (ret)
 909                 return ret;
 910
 911         /*
 912          * Now past the point of no return, this request _will_ be emitted.
 913          *
 914          * Or at least this preamble will be emitted, the request may be
 915          * interrupted prior to submitting the user payload. If so, we
 916          * still submit the "empty" request in order to preserve global
 917          * state tracking such as this, our tracking of the current
 918          * dirty context.
 919          */
 920         if (residuals) {
 921                 intel_context_put(*residuals);
 922                 *residuals = intel_context_get(ce);
 923         }
 924
 925         return 0;
 926 }
 927
 928 static int ring_request_alloc(struct i915_request *request)
 929 {
 930         int ret;
 931
 932         GEM_BUG_ON(!intel_context_is_pinned(request->context));
 933         GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb);
 934
 935         /*
 936          * Flush enough space to reduce the likelihood of waiting after
 937          * we start building the request - in which case we will just
 938          * have to repeat work.
 939          */
 940         request->reserved_space += LEGACY_REQUEST_SIZE;
 941
 942         /* Unconditionally invalidate GPU caches and TLBs. */
 943         ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
 944         if (ret)
 945                 return ret;
 946
 947         ret = switch_context(request);
 948         if (ret)
 949                 return ret;
 950
 951         request->reserved_space -= LEGACY_REQUEST_SIZE;
 952         return 0;
 953 }
 954
 955 static void gen6_bsd_submit_request(struct i915_request *request)
 956 {
 957         struct intel_uncore *uncore = request->engine->uncore;
 958
 959         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 960
 961        /* Every tail move must follow the sequence below */
 962
 963         /* Disable notification that the ring is IDLE. The GT
 964          * will then assume that it is busy and bring it out of rc6.
 965          */
 966         intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
 967                               _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
 968
 969         /* Clear the context id. Here be magic! */
 970         intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
 971
 972         /* Wait for the ring not to be idle, i.e. for it to wake up. */
 973         if (__intel_wait_for_register_fw(uncore,
 974                                          GEN6_BSD_SLEEP_PSMI_CONTROL,
 975                                          GEN6_BSD_SLEEP_INDICATOR,
 976                                          0,
 977                                          1000, 0, NULL))
 978                 drm_err(&uncore->i915->drm,
 979                         "timed out waiting for the BSD ring to wake up\n");
 980
 981         /* Now that the ring is fully powered up, update the tail */
 982         i9xx_submit_request(request);
 983
 984         /* Let the ring send IDLE messages to the GT again,
 985          * and so let it sleep to conserve power when idle.
 986          */
 987         intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
 988                               _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
 989
 990         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 991 }
 992
 993 static void i9xx_set_default_submission(struct intel_engine_cs *engine)
 994 {
 995         engine->submit_request = i9xx_submit_request;
 996 }
 997
 998 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
 999 {
1000         engine->submit_request = gen6_bsd_submit_request;
1001 }
1002
1003 static void ring_release(struct intel_engine_cs *engine)
1004 {
1005         struct drm_i915_private *dev_priv = engine->i915;
1006
1007         drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) > 2 &&
1008                     (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
1009
1010         intel_engine_cleanup_common(engine);
1011
1012         if (engine->wa_ctx.vma) {
1013                 intel_context_put(engine->wa_ctx.vma->private);
1014                 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
1015         }
1016
1017         intel_ring_unpin(engine->legacy.ring);
1018         intel_ring_put(engine->legacy.ring);
1019
1020         intel_timeline_unpin(engine->legacy.timeline);
1021         intel_timeline_put(engine->legacy.timeline);
1022 }
1023
1024 static void irq_handler(struct intel_engine_cs *engine, u16 iir)
1025 {
1026         intel_engine_signal_breadcrumbs(engine);
1027 }
1028
1029 static void setup_irq(struct intel_engine_cs *engine)
1030 {
1031         struct drm_i915_private *i915 = engine->i915;
1032
1033         intel_engine_set_irq_handler(engine, irq_handler);
1034
1035         if (GRAPHICS_VER(i915) >= 6) {
1036                 engine->irq_enable = gen6_irq_enable;
1037                 engine->irq_disable = gen6_irq_disable;
1038         } else if (GRAPHICS_VER(i915) >= 5) {
1039                 engine->irq_enable = gen5_irq_enable;
1040                 engine->irq_disable = gen5_irq_disable;
1041         } else if (GRAPHICS_VER(i915) >= 3) {
1042                 engine->irq_enable = gen3_irq_enable;
1043                 engine->irq_disable = gen3_irq_disable;
1044         } else {
1045                 engine->irq_enable = gen2_irq_enable;
1046                 engine->irq_disable = gen2_irq_disable;
1047         }
1048 }
1049
1050 static void setup_common(struct intel_engine_cs *engine)
1051 {
1052         struct drm_i915_private *i915 = engine->i915;
1053
1054         /* gen8+ are only supported with execlists */
1055         GEM_BUG_ON(GRAPHICS_VER(i915) >= 8);
1056
1057         setup_irq(engine);
1058
1059         engine->resume = xcs_resume;
1060         engine->sanitize = xcs_sanitize;
1061
1062         engine->reset.prepare = reset_prepare;
1063         engine->reset.rewind = reset_rewind;
1064         engine->reset.cancel = reset_cancel;
1065         engine->reset.finish = reset_finish;
1066
1067         engine->cops = &ring_context_ops;
1068         engine->request_alloc = ring_request_alloc;
1069
1070         /*
1071          * Using a global execution timeline; the previous final breadcrumb is
1072          * equivalent to our next initial bread so we can elide
1073          * engine->emit_init_breadcrumb().
1074          */
1075         engine->emit_fini_breadcrumb = gen3_emit_breadcrumb;
1076         if (GRAPHICS_VER(i915) == 5)
1077                 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb;
1078
1079         engine->set_default_submission = i9xx_set_default_submission;
1080
1081         if (GRAPHICS_VER(i915) >= 6)
1082                 engine->emit_bb_start = gen6_emit_bb_start;
1083         else if (GRAPHICS_VER(i915) >= 4)
1084                 engine->emit_bb_start = gen4_emit_bb_start;
1085         else if (IS_I830(i915) || IS_I845G(i915))
1086                 engine->emit_bb_start = i830_emit_bb_start;
1087         else
1088                 engine->emit_bb_start = gen3_emit_bb_start;
1089 }
1090
1091 static void setup_rcs(struct intel_engine_cs *engine)
1092 {
1093         struct drm_i915_private *i915 = engine->i915;
1094
1095         if (HAS_L3_DPF(i915))
1096                 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1097
1098         engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1099
1100         if (GRAPHICS_VER(i915) >= 7) {
1101                 engine->emit_flush = gen7_emit_flush_rcs;
1102                 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs;
1103         } else if (GRAPHICS_VER(i915) == 6) {
1104                 engine->emit_flush = gen6_emit_flush_rcs;
1105                 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs;
1106         } else if (GRAPHICS_VER(i915) == 5) {
1107                 engine->emit_flush = gen4_emit_flush_rcs;
1108         } else {
1109                 if (GRAPHICS_VER(i915) < 4)
1110                         engine->emit_flush = gen2_emit_flush;
1111                 else
1112                         engine->emit_flush = gen4_emit_flush_rcs;
1113                 engine->irq_enable_mask = I915_USER_INTERRUPT;
1114         }
1115
1116         if (IS_HASWELL(i915))
1117                 engine->emit_bb_start = hsw_emit_bb_start;
1118 }
1119
1120 static void setup_vcs(struct intel_engine_cs *engine)
1121 {
1122         struct drm_i915_private *i915 = engine->i915;
1123
1124         if (GRAPHICS_VER(i915) >= 6) {
1125                 /* gen6 bsd needs a special wa for tail updates */
1126                 if (GRAPHICS_VER(i915) == 6)
1127                         engine->set_default_submission = gen6_bsd_set_default_submission;
1128                 engine->emit_flush = gen6_emit_flush_vcs;
1129                 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1130
1131                 if (GRAPHICS_VER(i915) == 6)
1132                         engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
1133                 else
1134                         engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1135         } else {
1136                 engine->emit_flush = gen4_emit_flush_vcs;
1137                 if (GRAPHICS_VER(i915) == 5)
1138                         engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
1139                 else
1140                         engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1141         }
1142 }
1143
1144 static void setup_bcs(struct intel_engine_cs *engine)
1145 {
1146         struct drm_i915_private *i915 = engine->i915;
1147
1148         engine->emit_flush = gen6_emit_flush_xcs;
1149         engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
1150
1151         if (GRAPHICS_VER(i915) == 6)
1152                 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
1153         else
1154                 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1155 }
1156
1157 static void setup_vecs(struct intel_engine_cs *engine)
1158 {
1159         struct drm_i915_private *i915 = engine->i915;
1160
1161         GEM_BUG_ON(GRAPHICS_VER(i915) < 7);
1162
1163         engine->emit_flush = gen6_emit_flush_xcs;
1164         engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
1165         engine->irq_enable = hsw_irq_enable_vecs;
1166         engine->irq_disable = hsw_irq_disable_vecs;
1167
1168         engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1169 }
1170
1171 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine,
1172                                     struct i915_vma * const vma)
1173 {
1174         return gen7_setup_clear_gpr_bb(engine, vma);
1175 }
1176
1177 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine,
1178                                    struct i915_gem_ww_ctx *ww,
1179                                    struct i915_vma *vma)
1180 {
1181         int err;
1182
1183         err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH);
1184         if (err)
1185                 return err;
1186
1187         err = i915_vma_sync(vma);
1188         if (err)
1189                 goto err_unpin;
1190
1191         err = gen7_ctx_switch_bb_setup(engine, vma);
1192         if (err)
1193                 goto err_unpin;
1194
1195         engine->wa_ctx.vma = vma;
1196         return 0;
1197
1198 err_unpin:
1199         i915_vma_unpin(vma);
1200         return err;
1201 }
1202
1203 static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine)
1204 {
1205         struct drm_i915_gem_object *obj;
1206         struct i915_vma *vma;
1207         int size, err;
1208
1209         if (GRAPHICS_VER(engine->i915) != 7 || engine->class != RENDER_CLASS)
1210                 return 0;
1211
1212         err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
1213         if (err < 0)
1214                 return ERR_PTR(err);
1215         if (!err)
1216                 return NULL;
1217
1218         size = ALIGN(err, PAGE_SIZE);
1219
1220         obj = i915_gem_object_create_internal(engine->i915, size);
1221         if (IS_ERR(obj))
1222                 return ERR_CAST(obj);
1223
1224         vma = i915_vma_instance(obj, engine->gt->vm, NULL);
1225         if (IS_ERR(vma)) {
1226                 i915_gem_object_put(obj);
1227                 return ERR_CAST(vma);
1228         }
1229
1230         vma->private = intel_context_create(engine); /* dummy residuals */
1231         if (IS_ERR(vma->private)) {
1232                 err = PTR_ERR(vma->private);
1233                 vma->private = NULL;
1234                 i915_gem_object_put(obj);
1235                 return ERR_PTR(err);
1236         }
1237
1238         return vma;
1239 }
1240
1241 int intel_ring_submission_setup(struct intel_engine_cs *engine)
1242 {
1243         struct i915_gem_ww_ctx ww;
1244         struct intel_timeline *timeline;
1245         struct intel_ring *ring;
1246         struct i915_vma *gen7_wa_vma;
1247         int err;
1248
1249         setup_common(engine);
1250
1251         switch (engine->class) {
1252         case RENDER_CLASS:
1253                 setup_rcs(engine);
1254                 break;
1255         case VIDEO_DECODE_CLASS:
1256                 setup_vcs(engine);
1257                 break;
1258         case COPY_ENGINE_CLASS:
1259                 setup_bcs(engine);
1260                 break;
1261         case VIDEO_ENHANCEMENT_CLASS:
1262                 setup_vecs(engine);
1263                 break;
1264         default:
1265                 MISSING_CASE(engine->class);
1266                 return -ENODEV;
1267         }
1268
1269         timeline = intel_timeline_create_from_engine(engine,
1270                                                      I915_GEM_HWS_SEQNO_ADDR);
1271         if (IS_ERR(timeline)) {
1272                 err = PTR_ERR(timeline);
1273                 goto err;
1274         }
1275         GEM_BUG_ON(timeline->has_initial_breadcrumb);
1276
1277         ring = intel_engine_create_ring(engine, SZ_16K);
1278         if (IS_ERR(ring)) {
1279                 err = PTR_ERR(ring);
1280                 goto err_timeline;
1281         }
1282
1283         GEM_BUG_ON(engine->legacy.ring);
1284         engine->legacy.ring = ring;
1285         engine->legacy.timeline = timeline;
1286
1287         gen7_wa_vma = gen7_ctx_vma(engine);
1288         if (IS_ERR(gen7_wa_vma)) {
1289                 err = PTR_ERR(gen7_wa_vma);
1290                 goto err_ring;
1291         }
1292
1293         i915_gem_ww_ctx_init(&ww, false);
1294
1295 retry:
1296         err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww);
1297         if (!err && gen7_wa_vma)
1298                 err = i915_gem_object_lock(gen7_wa_vma->obj, &ww);
1299         if (!err && engine->legacy.ring->vma->obj)
1300                 err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww);
1301         if (!err)
1302                 err = intel_timeline_pin(timeline, &ww);
1303         if (!err) {
1304                 err = intel_ring_pin(ring, &ww);
1305                 if (err)
1306                         intel_timeline_unpin(timeline);
1307         }
1308         if (err)
1309                 goto out;
1310
1311         GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
1312
1313         if (gen7_wa_vma) {
1314                 err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma);
1315                 if (err) {
1316                         intel_ring_unpin(ring);
1317                         intel_timeline_unpin(timeline);
1318                 }
1319         }
1320
1321 out:
1322         if (err == -EDEADLK) {
1323                 err = i915_gem_ww_ctx_backoff(&ww);
1324                 if (!err)
1325                         goto retry;
1326         }
1327         i915_gem_ww_ctx_fini(&ww);
1328         if (err)
1329                 goto err_gen7_put;
1330
1331         /* Finally, take ownership and responsibility for cleanup! */
1332         engine->release = ring_release;
1333
1334         return 0;
1335
1336 err_gen7_put:
1337         if (gen7_wa_vma) {
1338                 intel_context_put(gen7_wa_vma->private);
1339                 i915_gem_object_put(gen7_wa_vma->obj);
1340         }
1341 err_ring:
1342         intel_ring_put(ring);
1343 err_timeline:
1344         intel_timeline_put(timeline);
1345 err:
1346         intel_engine_cleanup_common(engine);
1347         return err;
1348 }
1349
1350 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1351 #include "selftest_ring_submission.c"
1352 #endif