drivers/gpu/drm/i915/gt/intel_gt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_managed.h>
   7
   8 #include "gem/i915_gem_internal.h"
   9 #include "gem/i915_gem_lmem.h"
  10 #include "pxp/intel_pxp.h"
  11
  12 #include "i915_drv.h"
  13 #include "intel_context.h"
  14 #include "intel_engine_regs.h"
  15 #include "intel_gt.h"
  16 #include "intel_gt_buffer_pool.h"
  17 #include "intel_gt_clock_utils.h"
  18 #include "intel_gt_debugfs.h"
  19 #include "intel_gt_gmch.h"
  20 #include "intel_gt_pm.h"
  21 #include "intel_gt_regs.h"
  22 #include "intel_gt_requests.h"
  23 #include "intel_migrate.h"
  24 #include "intel_mocs.h"
  25 #include "intel_pm.h"
  26 #include "intel_rc6.h"
  27 #include "intel_renderstate.h"
  28 #include "intel_rps.h"
  29 #include "intel_gt_sysfs.h"
  30 #include "intel_uncore.h"
  31 #include "shmem_utils.h"
  32
  33 static void __intel_gt_init_early(struct intel_gt *gt)
  34 {
  35         spin_lock_init(&gt->irq_lock);
  36
  37         mutex_init(&gt->tlb_invalidate_lock);
  38
  39         INIT_LIST_HEAD(&gt->closed_vma);
  40         spin_lock_init(&gt->closed_lock);
  41
  42         init_llist_head(&gt->watchdog.list);
  43         INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work);
  44
  45         intel_gt_init_buffer_pool(gt);
  46         intel_gt_init_reset(gt);
  47         intel_gt_init_requests(gt);
  48         intel_gt_init_timelines(gt);
  49         intel_gt_pm_init_early(gt);
  50
  51         intel_uc_init_early(&gt->uc);
  52         intel_rps_init_early(&gt->rps);
  53 }
  54
  55 /* Preliminary initialization of Tile 0 */
  56 void intel_root_gt_init_early(struct drm_i915_private *i915)
  57 {
  58         struct intel_gt *gt = to_gt(i915);
  59
  60         gt->i915 = i915;
  61         gt->uncore = &i915->uncore;
  62
  63         __intel_gt_init_early(gt);
  64 }
  65
  66 static int intel_gt_probe_lmem(struct intel_gt *gt)
  67 {
  68         struct drm_i915_private *i915 = gt->i915;
  69         unsigned int instance = gt->info.id;
  70         int id = INTEL_REGION_LMEM_0 + instance;
  71         struct intel_memory_region *mem;
  72         int err;
  73
  74         mem = intel_gt_setup_lmem(gt);
  75         if (IS_ERR(mem)) {
  76                 err = PTR_ERR(mem);
  77                 if (err == -ENODEV)
  78                         return 0;
  79
  80                 drm_err(&i915->drm,
  81                         "Failed to setup region(%d) type=%d\n",
  82                         err, INTEL_MEMORY_LOCAL);
  83                 return err;
  84         }
  85
  86         mem->id = id;
  87         mem->instance = instance;
  88
  89         intel_memory_region_set_name(mem, "local%u", mem->instance);
  90
  91         GEM_BUG_ON(!HAS_REGION(i915, id));
  92         GEM_BUG_ON(i915->mm.regions[id]);
  93         i915->mm.regions[id] = mem;
  94
  95         return 0;
  96 }
  97
  98 int intel_gt_assign_ggtt(struct intel_gt *gt)
  99 {
 100         gt->ggtt = drmm_kzalloc(&gt->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL);
 101
 102         return gt->ggtt ? 0 : -ENOMEM;
 103 }
 104
 105 static const char * const intel_steering_types[] = {
 106         "L3BANK",
 107         "MSLICE",
 108         "LNCF",
 109 };
 110
 111 static const struct intel_mmio_range icl_l3bank_steering_table[] = {
 112         { 0x00B100, 0x00B3FF },
 113         {},
 114 };
 115
 116 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = {
 117         { 0x004000, 0x004AFF },
 118         { 0x00C800, 0x00CFFF },
 119         { 0x00DD00, 0x00DDFF },
 120         { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
 121         {},
 122 };
 123
 124 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = {
 125         { 0x00B000, 0x00B0FF },
 126         { 0x00D800, 0x00D8FF },
 127         {},
 128 };
 129
 130 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
 131         { 0x00B000, 0x00B0FF },
 132         { 0x00D880, 0x00D8FF },
 133         {},
 134 };
 135
 136 static u16 slicemask(struct intel_gt *gt, int count)
 137 {
 138         u64 dss_mask = intel_sseu_get_subslices(&gt->info.sseu, 0);
 139
 140         return intel_slicemask_from_dssmask(dss_mask, count);
 141 }
 142
 143 int intel_gt_init_mmio(struct intel_gt *gt)
 144 {
 145         struct drm_i915_private *i915 = gt->i915;
 146
 147         intel_gt_init_clock_frequency(gt);
 148
 149         intel_uc_init_mmio(&gt->uc);
 150         intel_sseu_info_init(gt);
 151
 152         /*
 153          * An mslice is unavailable only if both the meml3 for the slice is
 154          * disabled *and* all of the DSS in the slice (quadrant) are disabled.
 155          */
 156         if (HAS_MSLICES(i915))
 157                 gt->info.mslice_mask =
 158                         slicemask(gt, GEN_DSS_PER_MSLICE) |
 159                         (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
 160                          GEN12_MEML3_EN_MASK);
 161
 162         if (IS_DG2(i915)) {
 163                 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
 164                 gt->steering_table[LNCF] = dg2_lncf_steering_table;
 165         } else if (IS_XEHPSDV(i915)) {
 166                 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
 167                 gt->steering_table[LNCF] = xehpsdv_lncf_steering_table;
 168         } else if (GRAPHICS_VER(i915) >= 11 &&
 169                    GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) {
 170                 gt->steering_table[L3BANK] = icl_l3bank_steering_table;
 171                 gt->info.l3bank_mask =
 172                         ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
 173                         GEN10_L3BANK_MASK;
 174         } else if (HAS_MSLICES(i915)) {
 175                 MISSING_CASE(INTEL_INFO(i915)->platform);
 176         }
 177
 178         return intel_engines_init_mmio(gt);
 179 }
 180
 181 static void init_unused_ring(struct intel_gt *gt, u32 base)
 182 {
 183         struct intel_uncore *uncore = gt->uncore;
 184
 185         intel_uncore_write(uncore, RING_CTL(base), 0);
 186         intel_uncore_write(uncore, RING_HEAD(base), 0);
 187         intel_uncore_write(uncore, RING_TAIL(base), 0);
 188         intel_uncore_write(uncore, RING_START(base), 0);
 189 }
 190
 191 static void init_unused_rings(struct intel_gt *gt)
 192 {
 193         struct drm_i915_private *i915 = gt->i915;
 194
 195         if (IS_I830(i915)) {
 196                 init_unused_ring(gt, PRB1_BASE);
 197                 init_unused_ring(gt, SRB0_BASE);
 198                 init_unused_ring(gt, SRB1_BASE);
 199                 init_unused_ring(gt, SRB2_BASE);
 200                 init_unused_ring(gt, SRB3_BASE);
 201         } else if (GRAPHICS_VER(i915) == 2) {
 202                 init_unused_ring(gt, SRB0_BASE);
 203                 init_unused_ring(gt, SRB1_BASE);
 204         } else if (GRAPHICS_VER(i915) == 3) {
 205                 init_unused_ring(gt, PRB1_BASE);
 206                 init_unused_ring(gt, PRB2_BASE);
 207         }
 208 }
 209
 210 int intel_gt_init_hw(struct intel_gt *gt)
 211 {
 212         struct drm_i915_private *i915 = gt->i915;
 213         struct intel_uncore *uncore = gt->uncore;
 214         int ret;
 215
 216         gt->last_init_time = ktime_get();
 217
 218         /* Double layer security blanket, see i915_gem_init() */
 219         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 220
 221         if (HAS_EDRAM(i915) && GRAPHICS_VER(i915) < 9)
 222                 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
 223
 224         if (IS_HASWELL(i915))
 225                 intel_uncore_write(uncore,
 226                                    HSW_MI_PREDICATE_RESULT_2,
 227                                    IS_HSW_GT3(i915) ?
 228                                    LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 229
 230         /* Apply the GT workarounds... */
 231         intel_gt_apply_workarounds(gt);
 232         /* ...and determine whether they are sticking. */
 233         intel_gt_verify_workarounds(gt, "init");
 234
 235         intel_gt_init_swizzling(gt);
 236
 237         /*
 238          * At least 830 can leave some of the unused rings
 239          * "active" (ie. head != tail) after resume which
 240          * will prevent c3 entry. Makes sure all unused rings
 241          * are totally idle.
 242          */
 243         init_unused_rings(gt);
 244
 245         ret = i915_ppgtt_init_hw(gt);
 246         if (ret) {
 247                 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
 248                 goto out;
 249         }
 250
 251         /* We can't enable contexts until all firmware is loaded */
 252         ret = intel_uc_init_hw(&gt->uc);
 253         if (ret) {
 254                 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
 255                 goto out;
 256         }
 257
 258         intel_mocs_init(gt);
 259
 260 out:
 261         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 262         return ret;
 263 }
 264
 265 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
 266 {
 267         intel_uncore_rmw(uncore, reg, 0, set);
 268 }
 269
 270 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
 271 {
 272         intel_uncore_rmw(uncore, reg, clr, 0);
 273 }
 274
 275 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
 276 {
 277         intel_uncore_rmw(uncore, reg, 0, 0);
 278 }
 279
 280 static void gen6_clear_engine_error_register(struct intel_engine_cs *engine)
 281 {
 282         GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
 283         GEN6_RING_FAULT_REG_POSTING_READ(engine);
 284 }
 285
 286 void
 287 intel_gt_clear_error_registers(struct intel_gt *gt,
 288                                intel_engine_mask_t engine_mask)
 289 {
 290         struct drm_i915_private *i915 = gt->i915;
 291         struct intel_uncore *uncore = gt->uncore;
 292         u32 eir;
 293
 294         if (GRAPHICS_VER(i915) != 2)
 295                 clear_register(uncore, PGTBL_ER);
 296
 297         if (GRAPHICS_VER(i915) < 4)
 298                 clear_register(uncore, IPEIR(RENDER_RING_BASE));
 299         else
 300                 clear_register(uncore, IPEIR_I965);
 301
 302         clear_register(uncore, EIR);
 303         eir = intel_uncore_read(uncore, EIR);
 304         if (eir) {
 305                 /*
 306                  * some errors might have become stuck,
 307                  * mask them.
 308                  */
 309                 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
 310                 rmw_set(uncore, EMR, eir);
 311                 intel_uncore_write(uncore, GEN2_IIR,
 312                                    I915_MASTER_ERROR_INTERRUPT);
 313         }
 314
 315         if (GRAPHICS_VER(i915) >= 12) {
 316                 rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
 317                 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
 318         } else if (GRAPHICS_VER(i915) >= 8) {
 319                 rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
 320                 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
 321         } else if (GRAPHICS_VER(i915) >= 6) {
 322                 struct intel_engine_cs *engine;
 323                 enum intel_engine_id id;
 324
 325                 for_each_engine_masked(engine, gt, engine_mask, id)
 326                         gen6_clear_engine_error_register(engine);
 327         }
 328 }
 329
 330 static void gen6_check_faults(struct intel_gt *gt)
 331 {
 332         struct intel_engine_cs *engine;
 333         enum intel_engine_id id;
 334         u32 fault;
 335
 336         for_each_engine(engine, gt, id) {
 337                 fault = GEN6_RING_FAULT_REG_READ(engine);
 338                 if (fault & RING_FAULT_VALID) {
 339                         drm_dbg(&engine->i915->drm, "Unexpected fault\n"
 340                                 "\tAddr: 0x%08lx\n"
 341                                 "\tAddress space: %s\n"
 342                                 "\tSource ID: %d\n"
 343                                 "\tType: %d\n",
 344                                 fault & PAGE_MASK,
 345                                 fault & RING_FAULT_GTTSEL_MASK ?
 346                                 "GGTT" : "PPGTT",
 347                                 RING_FAULT_SRCID(fault),
 348                                 RING_FAULT_FAULT_TYPE(fault));
 349                 }
 350         }
 351 }
 352
 353 static void gen8_check_faults(struct intel_gt *gt)
 354 {
 355         struct intel_uncore *uncore = gt->uncore;
 356         i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
 357         u32 fault;
 358
 359         if (GRAPHICS_VER(gt->i915) >= 12) {
 360                 fault_reg = GEN12_RING_FAULT_REG;
 361                 fault_data0_reg = GEN12_FAULT_TLB_DATA0;
 362                 fault_data1_reg = GEN12_FAULT_TLB_DATA1;
 363         } else {
 364                 fault_reg = GEN8_RING_FAULT_REG;
 365                 fault_data0_reg = GEN8_FAULT_TLB_DATA0;
 366                 fault_data1_reg = GEN8_FAULT_TLB_DATA1;
 367         }
 368
 369         fault = intel_uncore_read(uncore, fault_reg);
 370         if (fault & RING_FAULT_VALID) {
 371                 u32 fault_data0, fault_data1;
 372                 u64 fault_addr;
 373
 374                 fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
 375                 fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
 376
 377                 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
 378                              ((u64)fault_data0 << 12);
 379
 380                 drm_dbg(&uncore->i915->drm, "Unexpected fault\n"
 381                         "\tAddr: 0x%08x_%08x\n"
 382                         "\tAddress space: %s\n"
 383                         "\tEngine ID: %d\n"
 384                         "\tSource ID: %d\n"
 385                         "\tType: %d\n",
 386                         upper_32_bits(fault_addr), lower_32_bits(fault_addr),
 387                         fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
 388                         GEN8_RING_FAULT_ENGINE_ID(fault),
 389                         RING_FAULT_SRCID(fault),
 390                         RING_FAULT_FAULT_TYPE(fault));
 391         }
 392 }
 393
 394 void intel_gt_check_and_clear_faults(struct intel_gt *gt)
 395 {
 396         struct drm_i915_private *i915 = gt->i915;
 397
 398         /* From GEN8 onwards we only have one 'All Engine Fault Register' */
 399         if (GRAPHICS_VER(i915) >= 8)
 400                 gen8_check_faults(gt);
 401         else if (GRAPHICS_VER(i915) >= 6)
 402                 gen6_check_faults(gt);
 403         else
 404                 return;
 405
 406         intel_gt_clear_error_registers(gt, ALL_ENGINES);
 407 }
 408
 409 void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
 410 {
 411         struct intel_uncore *uncore = gt->uncore;
 412         intel_wakeref_t wakeref;
 413
 414         /*
 415          * No actual flushing is required for the GTT write domain for reads
 416          * from the GTT domain. Writes to it "immediately" go to main memory
 417          * as far as we know, so there's no chipset flush. It also doesn't
 418          * land in the GPU render cache.
 419          *
 420          * However, we do have to enforce the order so that all writes through
 421          * the GTT land before any writes to the device, such as updates to
 422          * the GATT itself.
 423          *
 424          * We also have to wait a bit for the writes to land from the GTT.
 425          * An uncached read (i.e. mmio) seems to be ideal for the round-trip
 426          * timing. This issue has only been observed when switching quickly
 427          * between GTT writes and CPU reads from inside the kernel on recent hw,
 428          * and it appears to only affect discrete GTT blocks (i.e. on LLC
 429          * system agents we cannot reproduce this behaviour, until Cannonlake
 430          * that was!).
 431          */
 432
 433         wmb();
 434
 435         if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
 436                 return;
 437
 438         intel_gt_chipset_flush(gt);
 439
 440         with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
 441                 unsigned long flags;
 442
 443                 spin_lock_irqsave(&uncore->lock, flags);
 444                 intel_uncore_posting_read_fw(uncore,
 445                                              RING_HEAD(RENDER_RING_BASE));
 446                 spin_unlock_irqrestore(&uncore->lock, flags);
 447         }
 448 }
 449
 450 void intel_gt_chipset_flush(struct intel_gt *gt)
 451 {
 452         wmb();
 453         if (GRAPHICS_VER(gt->i915) < 6)
 454                 intel_gt_gmch_gen5_chipset_flush(gt);
 455 }
 456
 457 void intel_gt_driver_register(struct intel_gt *gt)
 458 {
 459         intel_gsc_init(&gt->gsc, gt->i915);
 460
 461         intel_rps_driver_register(&gt->rps);
 462
 463         intel_gt_debugfs_register(gt);
 464         intel_gt_sysfs_register(gt);
 465 }
 466
 467 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
 468 {
 469         struct drm_i915_private *i915 = gt->i915;
 470         struct drm_i915_gem_object *obj;
 471         struct i915_vma *vma;
 472         int ret;
 473
 474         obj = i915_gem_object_create_lmem(i915, size,
 475                                           I915_BO_ALLOC_VOLATILE |
 476                                           I915_BO_ALLOC_GPU_ONLY);
 477         if (IS_ERR(obj))
 478                 obj = i915_gem_object_create_stolen(i915, size);
 479         if (IS_ERR(obj))
 480                 obj = i915_gem_object_create_internal(i915, size);
 481         if (IS_ERR(obj)) {
 482                 drm_err(&i915->drm, "Failed to allocate scratch page\n");
 483                 return PTR_ERR(obj);
 484         }
 485
 486         vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
 487         if (IS_ERR(vma)) {
 488                 ret = PTR_ERR(vma);
 489                 goto err_unref;
 490         }
 491
 492         ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
 493         if (ret)
 494                 goto err_unref;
 495
 496         gt->scratch = i915_vma_make_unshrinkable(vma);
 497
 498         return 0;
 499
 500 err_unref:
 501         i915_gem_object_put(obj);
 502         return ret;
 503 }
 504
 505 static void intel_gt_fini_scratch(struct intel_gt *gt)
 506 {
 507         i915_vma_unpin_and_release(&gt->scratch, 0);
 508 }
 509
 510 static struct i915_address_space *kernel_vm(struct intel_gt *gt)
 511 {
 512         if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
 513                 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm;
 514         else
 515                 return i915_vm_get(&gt->ggtt->vm);
 516 }
 517
 518 static int __engines_record_defaults(struct intel_gt *gt)
 519 {
 520         struct i915_request *requests[I915_NUM_ENGINES] = {};
 521         struct intel_engine_cs *engine;
 522         enum intel_engine_id id;
 523         int err = 0;
 524
 525         /*
 526          * As we reset the gpu during very early sanitisation, the current
 527          * register state on the GPU should reflect its defaults values.
 528          * We load a context onto the hw (with restore-inhibit), then switch
 529          * over to a second context to save that default register state. We
 530          * can then prime every new context with that state so they all start
 531          * from the same default HW values.
 532          */
 533
 534         for_each_engine(engine, gt, id) {
 535                 struct intel_renderstate so;
 536                 struct intel_context *ce;
 537                 struct i915_request *rq;
 538
 539                 /* We must be able to switch to something! */
 540                 GEM_BUG_ON(!engine->kernel_context);
 541
 542                 ce = intel_context_create(engine);
 543                 if (IS_ERR(ce)) {
 544                         err = PTR_ERR(ce);
 545                         goto out;
 546                 }
 547
 548                 err = intel_renderstate_init(&so, ce);
 549                 if (err)
 550                         goto err;
 551
 552                 rq = i915_request_create(ce);
 553                 if (IS_ERR(rq)) {
 554                         err = PTR_ERR(rq);
 555                         goto err_fini;
 556                 }
 557
 558                 err = intel_engine_emit_ctx_wa(rq);
 559                 if (err)
 560                         goto err_rq;
 561
 562                 err = intel_renderstate_emit(&so, rq);
 563                 if (err)
 564                         goto err_rq;
 565
 566 err_rq:
 567                 requests[id] = i915_request_get(rq);
 568                 i915_request_add(rq);
 569 err_fini:
 570                 intel_renderstate_fini(&so, ce);
 571 err:
 572                 if (err) {
 573                         intel_context_put(ce);
 574                         goto out;
 575                 }
 576         }
 577
 578         /* Flush the default context image to memory, and enable powersaving. */
 579         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
 580                 err = -EIO;
 581                 goto out;
 582         }
 583
 584         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 585                 struct i915_request *rq;
 586                 struct file *state;
 587
 588                 rq = requests[id];
 589                 if (!rq)
 590                         continue;
 591
 592                 if (rq->fence.error) {
 593                         err = -EIO;
 594                         goto out;
 595                 }
 596
 597                 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
 598                 if (!rq->context->state)
 599                         continue;
 600
 601                 /* Keep a copy of the state's backing pages; free the obj */
 602                 state = shmem_create_from_object(rq->context->state->obj);
 603                 if (IS_ERR(state)) {
 604                         err = PTR_ERR(state);
 605                         goto out;
 606                 }
 607                 rq->engine->default_state = state;
 608         }
 609
 610 out:
 611         /*
 612          * If we have to abandon now, we expect the engines to be idle
 613          * and ready to be torn-down. The quickest way we can accomplish
 614          * this is by declaring ourselves wedged.
 615          */
 616         if (err)
 617                 intel_gt_set_wedged(gt);
 618
 619         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 620                 struct intel_context *ce;
 621                 struct i915_request *rq;
 622
 623                 rq = requests[id];
 624                 if (!rq)
 625                         continue;
 626
 627                 ce = rq->context;
 628                 i915_request_put(rq);
 629                 intel_context_put(ce);
 630         }
 631         return err;
 632 }
 633
 634 static int __engines_verify_workarounds(struct intel_gt *gt)
 635 {
 636         struct intel_engine_cs *engine;
 637         enum intel_engine_id id;
 638         int err = 0;
 639
 640         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 641                 return 0;
 642
 643         for_each_engine(engine, gt, id) {
 644                 if (intel_engine_verify_workarounds(engine, "load"))
 645                         err = -EIO;
 646         }
 647
 648         /* Flush and restore the kernel context for safety */
 649         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
 650                 err = -EIO;
 651
 652         return err;
 653 }
 654
 655 static void __intel_gt_disable(struct intel_gt *gt)
 656 {
 657         intel_gt_set_wedged_on_fini(gt);
 658
 659         intel_gt_suspend_prepare(gt);
 660         intel_gt_suspend_late(gt);
 661
 662         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 663 }
 664
 665 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
 666 {
 667         long remaining_timeout;
 668
 669         /* If the device is asleep, we have no requests outstanding */
 670         if (!intel_gt_pm_is_awake(gt))
 671                 return 0;
 672
 673         while ((timeout = intel_gt_retire_requests_timeout(gt, timeout,
 674                                                            &remaining_timeout)) > 0) {
 675                 cond_resched();
 676                 if (signal_pending(current))
 677                         return -EINTR;
 678         }
 679
 680         return timeout ? timeout : intel_uc_wait_for_idle(&gt->uc,
 681                                                           remaining_timeout);
 682 }
 683
 684 int intel_gt_init(struct intel_gt *gt)
 685 {
 686         int err;
 687
 688         err = i915_inject_probe_error(gt->i915, -ENODEV);
 689         if (err)
 690                 return err;
 691
 692         intel_gt_init_workarounds(gt);
 693
 694         /*
 695          * This is just a security blanket to placate dragons.
 696          * On some systems, we very sporadically observe that the first TLBs
 697          * used by the CS may be stale, despite us poking the TLB reset. If
 698          * we hold the forcewake during initialisation these problems
 699          * just magically go away.
 700          */
 701         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 702
 703         err = intel_gt_init_scratch(gt,
 704                                     GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K);
 705         if (err)
 706                 goto out_fw;
 707
 708         intel_gt_pm_init(gt);
 709
 710         gt->vm = kernel_vm(gt);
 711         if (!gt->vm) {
 712                 err = -ENOMEM;
 713                 goto err_pm;
 714         }
 715
 716         intel_set_mocs_index(gt);
 717
 718         err = intel_engines_init(gt);
 719         if (err)
 720                 goto err_engines;
 721
 722         err = intel_uc_init(&gt->uc);
 723         if (err)
 724                 goto err_engines;
 725
 726         err = intel_gt_resume(gt);
 727         if (err)
 728                 goto err_uc_init;
 729
 730         err = intel_gt_init_hwconfig(gt);
 731         if (err)
 732                 drm_err(&gt->i915->drm, "Failed to retrieve hwconfig table: %pe\n",
 733                         ERR_PTR(err));
 734
 735         err = __engines_record_defaults(gt);
 736         if (err)
 737                 goto err_gt;
 738
 739         err = __engines_verify_workarounds(gt);
 740         if (err)
 741                 goto err_gt;
 742
 743         intel_uc_init_late(&gt->uc);
 744
 745         err = i915_inject_probe_error(gt->i915, -EIO);
 746         if (err)
 747                 goto err_gt;
 748
 749         intel_migrate_init(&gt->migrate, gt);
 750
 751         intel_pxp_init(&gt->pxp);
 752
 753         goto out_fw;
 754 err_gt:
 755         __intel_gt_disable(gt);
 756         intel_uc_fini_hw(&gt->uc);
 757 err_uc_init:
 758         intel_uc_fini(&gt->uc);
 759 err_engines:
 760         intel_engines_release(gt);
 761         i915_vm_put(fetch_and_zero(&gt->vm));
 762 err_pm:
 763         intel_gt_pm_fini(gt);
 764         intel_gt_fini_scratch(gt);
 765 out_fw:
 766         if (err)
 767                 intel_gt_set_wedged_on_init(gt);
 768         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 769         return err;
 770 }
 771
 772 void intel_gt_driver_remove(struct intel_gt *gt)
 773 {
 774         __intel_gt_disable(gt);
 775
 776         intel_migrate_fini(&gt->migrate);
 777         intel_uc_driver_remove(&gt->uc);
 778
 779         intel_engines_release(gt);
 780
 781         intel_gt_flush_buffer_pool(gt);
 782 }
 783
 784 void intel_gt_driver_unregister(struct intel_gt *gt)
 785 {
 786         intel_wakeref_t wakeref;
 787
 788         intel_rps_driver_unregister(&gt->rps);
 789         intel_gsc_fini(&gt->gsc);
 790
 791         intel_pxp_fini(&gt->pxp);
 792
 793         /*
 794          * Upon unregistering the device to prevent any new users, cancel
 795          * all in-flight requests so that we can quickly unbind the active
 796          * resources.
 797          */
 798         intel_gt_set_wedged_on_fini(gt);
 799
 800         /* Scrub all HW state upon release */
 801         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
 802                 __intel_gt_reset(gt, ALL_ENGINES);
 803 }
 804
 805 void intel_gt_driver_release(struct intel_gt *gt)
 806 {
 807         struct i915_address_space *vm;
 808
 809         vm = fetch_and_zero(&gt->vm);
 810         if (vm) /* FIXME being called twice on error paths :( */
 811                 i915_vm_put(vm);
 812
 813         intel_wa_list_free(&gt->wa_list);
 814         intel_gt_pm_fini(gt);
 815         intel_gt_fini_scratch(gt);
 816         intel_gt_fini_buffer_pool(gt);
 817         intel_gt_fini_hwconfig(gt);
 818 }
 819
 820 void intel_gt_driver_late_release_all(struct drm_i915_private *i915)
 821 {
 822         struct intel_gt *gt;
 823         unsigned int id;
 824
 825         /* We need to wait for inflight RCU frees to release their grip */
 826         rcu_barrier();
 827
 828         for_each_gt(gt, i915, id) {
 829                 intel_uc_driver_late_release(&gt->uc);
 830                 intel_gt_fini_requests(gt);
 831                 intel_gt_fini_reset(gt);
 832                 intel_gt_fini_timelines(gt);
 833                 intel_engines_free(gt);
 834         }
 835 }
 836
 837 /**
 838  * intel_gt_reg_needs_read_steering - determine whether a register read
 839  *     requires explicit steering
 840  * @gt: GT structure
 841  * @reg: the register to check steering requirements for
 842  * @type: type of multicast steering to check
 843  *
 844  * Determines whether @reg needs explicit steering of a specific type for
 845  * reads.
 846  *
 847  * Returns false if @reg does not belong to a register range of the given
 848  * steering type, or if the default (subslice-based) steering IDs are suitable
 849  * for @type steering too.
 850  */
 851 static bool intel_gt_reg_needs_read_steering(struct intel_gt *gt,
 852                                              i915_reg_t reg,
 853                                              enum intel_steering_type type)
 854 {
 855         const u32 offset = i915_mmio_reg_offset(reg);
 856         const struct intel_mmio_range *entry;
 857
 858         if (likely(!intel_gt_needs_read_steering(gt, type)))
 859                 return false;
 860
 861         for (entry = gt->steering_table[type]; entry->end; entry++) {
 862                 if (offset >= entry->start && offset <= entry->end)
 863                         return true;
 864         }
 865
 866         return false;
 867 }
 868
 869 /**
 870  * intel_gt_get_valid_steering - determines valid IDs for a class of MCR steering
 871  * @gt: GT structure
 872  * @type: multicast register type
 873  * @sliceid: Slice ID returned
 874  * @subsliceid: Subslice ID returned
 875  *
 876  * Determines sliceid and subsliceid values that will steer reads
 877  * of a specific multicast register class to a valid value.
 878  */
 879 static void intel_gt_get_valid_steering(struct intel_gt *gt,
 880                                         enum intel_steering_type type,
 881                                         u8 *sliceid, u8 *subsliceid)
 882 {
 883         switch (type) {
 884         case L3BANK:
 885                 GEM_DEBUG_WARN_ON(!gt->info.l3bank_mask); /* should be impossible! */
 886
 887                 *sliceid = 0;           /* unused */
 888                 *subsliceid = __ffs(gt->info.l3bank_mask);
 889                 break;
 890         case MSLICE:
 891                 GEM_DEBUG_WARN_ON(!gt->info.mslice_mask); /* should be impossible! */
 892
 893                 *sliceid = __ffs(gt->info.mslice_mask);
 894                 *subsliceid = 0;        /* unused */
 895                 break;
 896         case LNCF:
 897                 GEM_DEBUG_WARN_ON(!gt->info.mslice_mask); /* should be impossible! */
 898
 899                 /*
 900                  * An LNCF is always present if its mslice is present, so we
 901                  * can safely just steer to LNCF 0 in all cases.
 902                  */
 903                 *sliceid = __ffs(gt->info.mslice_mask) << 1;
 904                 *subsliceid = 0;        /* unused */
 905                 break;
 906         default:
 907                 MISSING_CASE(type);
 908                 *sliceid = 0;
 909                 *subsliceid = 0;
 910         }
 911 }
 912
 913 /**
 914  * intel_gt_read_register_fw - reads a GT register with support for multicast
 915  * @gt: GT structure
 916  * @reg: register to read
 917  *
 918  * This function will read a GT register.  If the register is a multicast
 919  * register, the read will be steered to a valid instance (i.e., one that
 920  * isn't fused off or powered down by power gating).
 921  *
 922  * Returns the value from a valid instance of @reg.
 923  */
 924 u32 intel_gt_read_register_fw(struct intel_gt *gt, i915_reg_t reg)
 925 {
 926         int type;
 927         u8 sliceid, subsliceid;
 928
 929         for (type = 0; type < NUM_STEERING_TYPES; type++) {
 930                 if (intel_gt_reg_needs_read_steering(gt, reg, type)) {
 931                         intel_gt_get_valid_steering(gt, type, &sliceid,
 932                                                     &subsliceid);
 933                         return intel_uncore_read_with_mcr_steering_fw(gt->uncore,
 934                                                                       reg,
 935                                                                       sliceid,
 936                                                                       subsliceid);
 937                 }
 938         }
 939
 940         return intel_uncore_read_fw(gt->uncore, reg);
 941 }
 942
 943 /**
 944  * intel_gt_get_valid_steering_for_reg - get a valid steering for a register
 945  * @gt: GT structure
 946  * @reg: register for which the steering is required
 947  * @sliceid: return variable for slice steering
 948  * @subsliceid: return variable for subslice steering
 949  *
 950  * This function returns a slice/subslice pair that is guaranteed to work for
 951  * read steering of the given register. Note that a value will be returned even
 952  * if the register is not replicated and therefore does not actually require
 953  * steering.
 954  */
 955 void intel_gt_get_valid_steering_for_reg(struct intel_gt *gt, i915_reg_t reg,
 956                                          u8 *sliceid, u8 *subsliceid)
 957 {
 958         int type;
 959
 960         for (type = 0; type < NUM_STEERING_TYPES; type++) {
 961                 if (intel_gt_reg_needs_read_steering(gt, reg, type)) {
 962                         intel_gt_get_valid_steering(gt, type, sliceid,
 963                                                     subsliceid);
 964                         return;
 965                 }
 966         }
 967
 968         *sliceid = gt->default_steering.groupid;
 969         *subsliceid = gt->default_steering.instanceid;
 970 }
 971
 972 u32 intel_gt_read_register(struct intel_gt *gt, i915_reg_t reg)
 973 {
 974         int type;
 975         u8 sliceid, subsliceid;
 976
 977         for (type = 0; type < NUM_STEERING_TYPES; type++) {
 978                 if (intel_gt_reg_needs_read_steering(gt, reg, type)) {
 979                         intel_gt_get_valid_steering(gt, type, &sliceid,
 980                                                     &subsliceid);
 981                         return intel_uncore_read_with_mcr_steering(gt->uncore,
 982                                                                    reg,
 983                                                                    sliceid,
 984                                                                    subsliceid);
 985                 }
 986         }
 987
 988         return intel_uncore_read(gt->uncore, reg);
 989 }
 990
 991 static void report_steering_type(struct drm_printer *p,
 992                                  struct intel_gt *gt,
 993                                  enum intel_steering_type type,
 994                                  bool dump_table)
 995 {
 996         const struct intel_mmio_range *entry;
 997         u8 slice, subslice;
 998
 999         BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
1000
1001         if (!gt->steering_table[type]) {
1002                 drm_printf(p, "%s steering: uses default steering\n",
1003                            intel_steering_types[type]);
1004                 return;
1005         }
1006
1007         intel_gt_get_valid_steering(gt, type, &slice, &subslice);
1008         drm_printf(p, "%s steering: sliceid=0x%x, subsliceid=0x%x\n",
1009                    intel_steering_types[type], slice, subslice);
1010
1011         if (!dump_table)
1012                 return;
1013
1014         for (entry = gt->steering_table[type]; entry->end; entry++)
1015                 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);
1016 }
1017
1018 void intel_gt_report_steering(struct drm_printer *p, struct intel_gt *gt,
1019                               bool dump_table)
1020 {
1021         drm_printf(p, "Default steering: sliceid=0x%x, subsliceid=0x%x\n",
1022                    gt->default_steering.groupid,
1023                    gt->default_steering.instanceid);
1024
1025         if (HAS_MSLICES(gt->i915)) {
1026                 report_steering_type(p, gt, MSLICE, dump_table);
1027                 report_steering_type(p, gt, LNCF, dump_table);
1028         }
1029 }
1030
1031 static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)
1032 {
1033         int ret;
1034
1035         if (!gt_is_root(gt)) {
1036                 struct intel_uncore_mmio_debug *mmio_debug;
1037                 struct intel_uncore *uncore;
1038
1039                 uncore = kzalloc(sizeof(*uncore), GFP_KERNEL);
1040                 if (!uncore)
1041                         return -ENOMEM;
1042
1043                 mmio_debug = kzalloc(sizeof(*mmio_debug), GFP_KERNEL);
1044                 if (!mmio_debug) {
1045                         kfree(uncore);
1046                         return -ENOMEM;
1047                 }
1048
1049                 gt->uncore = uncore;
1050                 gt->uncore->debug = mmio_debug;
1051
1052                 __intel_gt_init_early(gt);
1053         }
1054
1055         intel_uncore_init_early(gt->uncore, gt);
1056
1057         ret = intel_uncore_setup_mmio(gt->uncore, phys_addr);
1058         if (ret)
1059                 return ret;
1060
1061         gt->phys_addr = phys_addr;
1062
1063         return 0;
1064 }
1065
1066 static void
1067 intel_gt_tile_cleanup(struct intel_gt *gt)
1068 {
1069         intel_uncore_cleanup_mmio(gt->uncore);
1070
1071         if (!gt_is_root(gt)) {
1072                 kfree(gt->uncore->debug);
1073                 kfree(gt->uncore);
1074                 kfree(gt);
1075         }
1076 }
1077
1078 int intel_gt_probe_all(struct drm_i915_private *i915)
1079 {
1080         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1081         struct intel_gt *gt = &i915->gt0;
1082         phys_addr_t phys_addr;
1083         unsigned int mmio_bar;
1084         int ret;
1085
1086         mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0;
1087         phys_addr = pci_resource_start(pdev, mmio_bar);
1088
1089         /*
1090          * We always have at least one primary GT on any device
1091          * and it has been already initialized early during probe
1092          * in i915_driver_probe()
1093          */
1094         ret = intel_gt_tile_setup(gt, phys_addr);
1095         if (ret)
1096                 return ret;
1097
1098         i915->gt[0] = gt;
1099
1100         /* TODO: add more tiles */
1101         return 0;
1102 }
1103
1104 int intel_gt_tiles_init(struct drm_i915_private *i915)
1105 {
1106         struct intel_gt *gt;
1107         unsigned int id;
1108         int ret;
1109
1110         for_each_gt(gt, i915, id) {
1111                 ret = intel_gt_probe_lmem(gt);
1112                 if (ret)
1113                         return ret;
1114         }
1115
1116         return 0;
1117 }
1118
1119 void intel_gt_release_all(struct drm_i915_private *i915)
1120 {
1121         struct intel_gt *gt;
1122         unsigned int id;
1123
1124         for_each_gt(gt, i915, id) {
1125                 intel_gt_tile_cleanup(gt);
1126                 i915->gt[id] = NULL;
1127         }
1128 }
1129
1130 void intel_gt_info_print(const struct intel_gt_info *info,
1131                          struct drm_printer *p)
1132 {
1133         drm_printf(p, "available engines: %x\n", info->engine_mask);
1134
1135         intel_sseu_dump(&info->sseu, p);
1136 }
1137
1138 struct reg_and_bit {
1139         i915_reg_t reg;
1140         u32 bit;
1141 };
1142
1143 static struct reg_and_bit
1144 get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
1145                 const i915_reg_t *regs, const unsigned int num)
1146 {
1147         const unsigned int class = engine->class;
1148         struct reg_and_bit rb = { };
1149
1150         if (drm_WARN_ON_ONCE(&engine->i915->drm,
1151                              class >= num || !regs[class].reg))
1152                 return rb;
1153
1154         rb.reg = regs[class];
1155         if (gen8 && class == VIDEO_DECODE_CLASS)
1156                 rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */
1157         else
1158                 rb.bit = engine->instance;
1159
1160         rb.bit = BIT(rb.bit);
1161
1162         return rb;
1163 }
1164
1165 void intel_gt_invalidate_tlbs(struct intel_gt *gt)
1166 {
1167         static const i915_reg_t gen8_regs[] = {
1168                 [RENDER_CLASS]                  = GEN8_RTCR,
1169                 [VIDEO_DECODE_CLASS]            = GEN8_M1TCR, /* , GEN8_M2TCR */
1170                 [VIDEO_ENHANCEMENT_CLASS]       = GEN8_VTCR,
1171                 [COPY_ENGINE_CLASS]             = GEN8_BTCR,
1172         };
1173         static const i915_reg_t gen12_regs[] = {
1174                 [RENDER_CLASS]                  = GEN12_GFX_TLB_INV_CR,
1175                 [VIDEO_DECODE_CLASS]            = GEN12_VD_TLB_INV_CR,
1176                 [VIDEO_ENHANCEMENT_CLASS]       = GEN12_VE_TLB_INV_CR,
1177                 [COPY_ENGINE_CLASS]             = GEN12_BLT_TLB_INV_CR,
1178                 [COMPUTE_CLASS]                 = GEN12_COMPCTX_TLB_INV_CR,
1179         };
1180         struct drm_i915_private *i915 = gt->i915;
1181         struct intel_uncore *uncore = gt->uncore;
1182         struct intel_engine_cs *engine;
1183         enum intel_engine_id id;
1184         const i915_reg_t *regs;
1185         unsigned int num = 0;
1186
1187         if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
1188                 return;
1189
1190         if (GRAPHICS_VER(i915) == 12) {
1191                 regs = gen12_regs;
1192                 num = ARRAY_SIZE(gen12_regs);
1193         } else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) {
1194                 regs = gen8_regs;
1195                 num = ARRAY_SIZE(gen8_regs);
1196         } else if (GRAPHICS_VER(i915) < 8) {
1197                 return;
1198         }
1199
1200         if (drm_WARN_ONCE(&i915->drm, !num,
1201                           "Platform does not implement TLB invalidation!"))
1202                 return;
1203
1204         GEM_TRACE("\n");
1205
1206         assert_rpm_wakelock_held(&i915->runtime_pm);
1207
1208         mutex_lock(&gt->tlb_invalidate_lock);
1209         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1210
1211         for_each_engine(engine, gt, id) {
1212                 /*
1213                  * HW architecture suggest typical invalidation time at 40us,
1214                  * with pessimistic cases up to 100us and a recommendation to
1215                  * cap at 1ms. We go a bit higher just in case.
1216                  */
1217                 const unsigned int timeout_us = 100;
1218                 const unsigned int timeout_ms = 4;
1219                 struct reg_and_bit rb;
1220
1221                 rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
1222                 if (!i915_mmio_reg_offset(rb.reg))
1223                         continue;
1224
1225                 intel_uncore_write_fw(uncore, rb.reg, rb.bit);
1226                 if (__intel_wait_for_register_fw(uncore,
1227                                                  rb.reg, rb.bit, 0,
1228                                                  timeout_us, timeout_ms,
1229                                                  NULL))
1230                         drm_err_ratelimited(&gt->i915->drm,
1231                                             "%s TLB invalidation did not complete in %ums!\n",
1232                                             engine->name, timeout_ms);
1233         }
1234
1235         /*
1236          * Use delayed put since a) we mostly expect a flurry of TLB
1237          * invalidations so it is good to avoid paying the forcewake cost and
1238          * b) it works around a bug in Icelake which cannot cope with too rapid
1239          * transitions.
1240          */
1241         intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL);
1242         mutex_unlock(&gt->tlb_invalidate_lock);
1243 }