drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_engine_pm.h"
  10 #include "intel_gt.h"
  11 #include "intel_ring.h"
  12 #include "intel_workarounds.h"
  13
  14 /**
  15  * DOC: Hardware workarounds
  16  *
  17  * This file is intended as a central place to implement most [1]_ of the
  18  * required workarounds for hardware to work as originally intended. They fall
  19  * in five basic categories depending on how/when they are applied:
  20  *
  21  * - Workarounds that touch registers that are saved/restored to/from the HW
  22  *   context image. The list is emitted (via Load Register Immediate commands)
  23  *   everytime a new context is created.
  24  * - GT workarounds. The list of these WAs is applied whenever these registers
  25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  26  * - Display workarounds. The list is applied during display clock-gating
  27  *   initialization.
  28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  29  *   them directly. This is just a special case of a MMMIO workaround (as we
  30  *   write the list of these to/be-whitelisted registers to some special HW
  31  *   registers).
  32  * - Workaround batchbuffers, that get executed automatically by the hardware
  33  *   on every HW context restore.
  34  *
  35  * .. [1] Please notice that there are other WAs that, due to their nature,
  36  *    cannot be applied from a central place. Those are peppered around the rest
  37  *    of the code, as needed.
  38  *
  39  * .. [2] Technically, some registers are powercontext saved & restored, so they
  40  *    survive a suspend/resume. In practice, writing them again is not too
  41  *    costly and simplifies things. We can revisit this in the future.
  42  *
  43  * Layout
  44  * ~~~~~~
  45  *
  46  * Keep things in this file ordered by WA type, as per the above (context, GT,
  47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  48  * following order:
  49  *
  50  * - Infrastructure functions and macros
  51  * - WAs per platform in standard gen/chrono order
  52  * - Public functions to init or apply the given workaround type.
  53  */
  54
  55 /*
  56  * KBL revision ID ordering is bizarre; higher revision ID's map to lower
  57  * steppings in some cases.  So rather than test against the revision ID
  58  * directly, let's map that into our own range of increasing ID's that we
  59  * can test against in a regular manner.
  60  */
  61
  62 const struct i915_rev_steppings kbl_revids[] = {
  63         [0] = { .gt_stepping = KBL_REVID_A0, .disp_stepping = KBL_REVID_A0 },
  64         [1] = { .gt_stepping = KBL_REVID_B0, .disp_stepping = KBL_REVID_B0 },
  65         [2] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B0 },
  66         [3] = { .gt_stepping = KBL_REVID_D0, .disp_stepping = KBL_REVID_B0 },
  67         [4] = { .gt_stepping = KBL_REVID_F0, .disp_stepping = KBL_REVID_C0 },
  68         [5] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B1 },
  69         [6] = { .gt_stepping = KBL_REVID_D1, .disp_stepping = KBL_REVID_B1 },
  70         [7] = { .gt_stepping = KBL_REVID_G0, .disp_stepping = KBL_REVID_C0 },
  71 };
  72
  73 const struct i915_rev_steppings tgl_uy_revids[] = {
  74         [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_A0 },
  75         [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_C0 },
  76         [2] = { .gt_stepping = TGL_REVID_B1, .disp_stepping = TGL_REVID_C0 },
  77         [3] = { .gt_stepping = TGL_REVID_C0, .disp_stepping = TGL_REVID_D0 },
  78 };
  79
  80 /* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */
  81 const struct i915_rev_steppings tgl_revids[] = {
  82         [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_B0 },
  83         [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_D0 },
  84 };
  85
  86 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  87 {
  88         wal->name = name;
  89         wal->engine_name = engine_name;
  90 }
  91
  92 #define WA_LIST_CHUNK (1 << 4)
  93
  94 static void wa_init_finish(struct i915_wa_list *wal)
  95 {
  96         /* Trim unused entries. */
  97         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  98                 struct i915_wa *list = kmemdup(wal->list,
  99                                                wal->count * sizeof(*list),
 100                                                GFP_KERNEL);
 101
 102                 if (list) {
 103                         kfree(wal->list);
 104                         wal->list = list;
 105                 }
 106         }
 107
 108         if (!wal->count)
 109                 return;
 110
 111         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
 112                          wal->wa_count, wal->name, wal->engine_name);
 113 }
 114
 115 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
 116 {
 117         unsigned int addr = i915_mmio_reg_offset(wa->reg);
 118         unsigned int start = 0, end = wal->count;
 119         const unsigned int grow = WA_LIST_CHUNK;
 120         struct i915_wa *wa_;
 121
 122         GEM_BUG_ON(!is_power_of_2(grow));
 123
 124         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
 125                 struct i915_wa *list;
 126
 127                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
 128                                      GFP_KERNEL);
 129                 if (!list) {
 130                         DRM_ERROR("No space for workaround init!\n");
 131                         return;
 132                 }
 133
 134                 if (wal->list) {
 135                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 136                         kfree(wal->list);
 137                 }
 138
 139                 wal->list = list;
 140         }
 141
 142         while (start < end) {
 143                 unsigned int mid = start + (end - start) / 2;
 144
 145                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 146                         start = mid + 1;
 147                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 148                         end = mid;
 149                 } else {
 150                         wa_ = &wal->list[mid];
 151
 152                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 153                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 154                                           i915_mmio_reg_offset(wa_->reg),
 155                                           wa_->clr, wa_->set);
 156
 157                                 wa_->set &= ~wa->clr;
 158                         }
 159
 160                         wal->wa_count++;
 161                         wa_->set |= wa->set;
 162                         wa_->clr |= wa->clr;
 163                         wa_->read |= wa->read;
 164                         return;
 165                 }
 166         }
 167
 168         wal->wa_count++;
 169         wa_ = &wal->list[wal->count++];
 170         *wa_ = *wa;
 171
 172         while (wa_-- > wal->list) {
 173                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 174                            i915_mmio_reg_offset(wa_[1].reg));
 175                 if (i915_mmio_reg_offset(wa_[1].reg) >
 176                     i915_mmio_reg_offset(wa_[0].reg))
 177                         break;
 178
 179                 swap(wa_[1], wa_[0]);
 180         }
 181 }
 182
 183 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 184                    u32 clear, u32 set, u32 read_mask)
 185 {
 186         struct i915_wa wa = {
 187                 .reg  = reg,
 188                 .clr  = clear,
 189                 .set  = set,
 190                 .read = read_mask,
 191         };
 192
 193         _wa_add(wal, &wa);
 194 }
 195
 196 static void
 197 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 198 {
 199         wa_add(wal, reg, clear, set, clear);
 200 }
 201
 202 static void
 203 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 204 {
 205         wa_write_masked_or(wal, reg, ~0, set);
 206 }
 207
 208 static void
 209 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 210 {
 211         wa_write_masked_or(wal, reg, set, set);
 212 }
 213
 214 static void
 215 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
 216 {
 217         wa_write_masked_or(wal, reg, clr, 0);
 218 }
 219
 220 static void
 221 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 222 {
 223         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
 224 }
 225
 226 static void
 227 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 228 {
 229         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
 230 }
 231
 232 #define WA_SET_BIT_MASKED(addr, mask) \
 233         wa_masked_en(wal, (addr), (mask))
 234
 235 #define WA_CLR_BIT_MASKED(addr, mask) \
 236         wa_masked_dis(wal, (addr), (mask))
 237
 238 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 239         wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
 240
 241 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
 242                                       struct i915_wa_list *wal)
 243 {
 244         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 245 }
 246
 247 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
 248                                       struct i915_wa_list *wal)
 249 {
 250         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 251 }
 252
 253 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 254                                       struct i915_wa_list *wal)
 255 {
 256         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 257
 258         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 259         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 260
 261         /* WaDisablePartialInstShootdown:bdw,chv */
 262         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 263                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 264
 265         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 266          * workaround for for a possible hang in the unlikely event a TLB
 267          * invalidation occurs during a PSD flush.
 268          */
 269         /* WaForceEnableNonCoherent:bdw,chv */
 270         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 271         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 272                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 273                           HDC_FORCE_NON_COHERENT);
 274
 275         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 276          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 277          *  polygons in the same 8x4 pixel/sample area to be processed without
 278          *  stalling waiting for the earlier ones to write to Hierarchical Z
 279          *  buffer."
 280          *
 281          * This optimization is off by default for BDW and CHV; turn it on.
 282          */
 283         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 284
 285         /* Wa4x4STCOptimizationDisable:bdw,chv */
 286         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 287
 288         /*
 289          * BSpec recommends 8x4 when MSAA is used,
 290          * however in practice 16x4 seems fastest.
 291          *
 292          * Note that PS/WM thread counts depend on the WIZ hashing
 293          * disable bit, which we don't touch here, but it's good
 294          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 295          */
 296         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 297                             GEN6_WIZ_HASHING_MASK,
 298                             GEN6_WIZ_HASHING_16x4);
 299 }
 300
 301 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 302                                      struct i915_wa_list *wal)
 303 {
 304         struct drm_i915_private *i915 = engine->i915;
 305
 306         gen8_ctx_workarounds_init(engine, wal);
 307
 308         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 309         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 310
 311         /* WaDisableDopClockGating:bdw
 312          *
 313          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 314          * to disable EUTC clock gating.
 315          */
 316         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 317                           DOP_CLOCK_GATING_DISABLE);
 318
 319         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 320                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 321
 322         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 323                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 324                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 325                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 326                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 327 }
 328
 329 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 330                                      struct i915_wa_list *wal)
 331 {
 332         gen8_ctx_workarounds_init(engine, wal);
 333
 334         /* WaDisableThreadStallDopClockGating:chv */
 335         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 336
 337         /* Improve HiZ throughput on CHV. */
 338         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 339 }
 340
 341 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 342                                       struct i915_wa_list *wal)
 343 {
 344         struct drm_i915_private *i915 = engine->i915;
 345
 346         if (HAS_LLC(i915)) {
 347                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 348                  *
 349                  * Must match Display Engine. See
 350                  * WaCompressedResourceDisplayNewHashMode.
 351                  */
 352                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 353                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 354                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 355                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 356         }
 357
 358         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 359         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 360         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 361                           FLOW_CONTROL_ENABLE |
 362                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 363
 364         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 365         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 366         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 367                           GEN9_ENABLE_YV12_BUGFIX |
 368                           GEN9_ENABLE_GPGPU_PREEMPTION);
 369
 370         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 371         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 372         WA_SET_BIT_MASKED(CACHE_MODE_1,
 373                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 374                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 375
 376         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 377         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 378                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 379
 380         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 381         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 382                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 383                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 384
 385         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 386          * both tied to WaForceContextSaveRestoreNonCoherent
 387          * in some hsds for skl. We keep the tie for all gen9. The
 388          * documentation is a bit hazy and so we want to get common behaviour,
 389          * even though there is no clear evidence we would need both on kbl/bxt.
 390          * This area has been source of system hangs so we play it safe
 391          * and mimic the skl regardless of what bspec says.
 392          *
 393          * Use Force Non-Coherent whenever executing a 3D context. This
 394          * is a workaround for a possible hang in the unlikely event
 395          * a TLB invalidation occurs during a PSD flush.
 396          */
 397
 398         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 399         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 400                           HDC_FORCE_NON_COHERENT);
 401
 402         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 403         if (IS_SKYLAKE(i915) ||
 404             IS_KABYLAKE(i915) ||
 405             IS_COFFEELAKE(i915) ||
 406             IS_COMETLAKE(i915))
 407                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 408                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 409
 410         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 411         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 412
 413         /*
 414          * Supporting preemption with fine-granularity requires changes in the
 415          * batch buffer programming. Since we can't break old userspace, we
 416          * need to set our default preemption level to safe value. Userspace is
 417          * still able to use more fine-grained preemption levels, since in
 418          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 419          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 420          * not real HW workarounds, but merely a way to start using preemption
 421          * while maintaining old contract with userspace.
 422          */
 423
 424         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 425         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 426
 427         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 428         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 429                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 430                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 431
 432         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 433         if (IS_GEN9_LP(i915))
 434                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 435 }
 436
 437 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 438                                 struct i915_wa_list *wal)
 439 {
 440         struct intel_gt *gt = engine->gt;
 441         u8 vals[3] = { 0, 0, 0 };
 442         unsigned int i;
 443
 444         for (i = 0; i < 3; i++) {
 445                 u8 ss;
 446
 447                 /*
 448                  * Only consider slices where one, and only one, subslice has 7
 449                  * EUs
 450                  */
 451                 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
 452                         continue;
 453
 454                 /*
 455                  * subslice_7eu[i] != 0 (because of the check above) and
 456                  * ss_max == 4 (maximum number of subslices possible per slice)
 457                  *
 458                  * ->    0 <= ss <= 3;
 459                  */
 460                 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
 461                 vals[i] = 3 - ss;
 462         }
 463
 464         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 465                 return;
 466
 467         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 468         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 469                             GEN9_IZ_HASHING_MASK(2) |
 470                             GEN9_IZ_HASHING_MASK(1) |
 471                             GEN9_IZ_HASHING_MASK(0),
 472                             GEN9_IZ_HASHING(2, vals[2]) |
 473                             GEN9_IZ_HASHING(1, vals[1]) |
 474                             GEN9_IZ_HASHING(0, vals[0]));
 475 }
 476
 477 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 478                                      struct i915_wa_list *wal)
 479 {
 480         gen9_ctx_workarounds_init(engine, wal);
 481         skl_tune_iz_hashing(engine, wal);
 482 }
 483
 484 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 485                                      struct i915_wa_list *wal)
 486 {
 487         gen9_ctx_workarounds_init(engine, wal);
 488
 489         /* WaDisableThreadStallDopClockGating:bxt */
 490         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 491                           STALL_DOP_GATING_DISABLE);
 492
 493         /* WaToEnableHwFixForPushConstHWBug:bxt */
 494         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 495                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 496 }
 497
 498 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 499                                      struct i915_wa_list *wal)
 500 {
 501         struct drm_i915_private *i915 = engine->i915;
 502
 503         gen9_ctx_workarounds_init(engine, wal);
 504
 505         /* WaToEnableHwFixForPushConstHWBug:kbl */
 506         if (IS_KBL_GT_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 507                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 508                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 509
 510         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 511         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 512                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 513 }
 514
 515 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 516                                      struct i915_wa_list *wal)
 517 {
 518         gen9_ctx_workarounds_init(engine, wal);
 519
 520         /* WaToEnableHwFixForPushConstHWBug:glk */
 521         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 522                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 523 }
 524
 525 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 526                                      struct i915_wa_list *wal)
 527 {
 528         gen9_ctx_workarounds_init(engine, wal);
 529
 530         /* WaToEnableHwFixForPushConstHWBug:cfl */
 531         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 532                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 533
 534         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 535         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 536                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 537 }
 538
 539 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 540                                      struct i915_wa_list *wal)
 541 {
 542         /* WaForceContextSaveRestoreNonCoherent:cnl */
 543         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 544                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 545
 546         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 547         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 548                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 549
 550         /* WaPushConstantDereferenceHoldDisable:cnl */
 551         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 552
 553         /* FtrEnableFastAnisoL1BankingFix:cnl */
 554         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 555
 556         /* WaDisable3DMidCmdPreemption:cnl */
 557         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 558
 559         /* WaDisableGPGPUMidCmdPreemption:cnl */
 560         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 561                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 562                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 563
 564         /* WaDisableEarlyEOT:cnl */
 565         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 566 }
 567
 568 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 569                                      struct i915_wa_list *wal)
 570 {
 571         struct drm_i915_private *i915 = engine->i915;
 572
 573         /* WaDisableBankHangMode:icl */
 574         wa_write(wal,
 575                  GEN8_L3CNTLREG,
 576                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 577                  GEN8_ERRDETBCTRL);
 578
 579         /* Wa_1604370585:icl (pre-prod)
 580          * Formerly known as WaPushConstantDereferenceHoldDisable
 581          */
 582         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 583                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 584                                   PUSH_CONSTANT_DEREF_DISABLE);
 585
 586         /* WaForceEnableNonCoherent:icl
 587          * This is not the same workaround as in early Gen9 platforms, where
 588          * lacking this could cause system hangs, but coherency performance
 589          * overhead is high and only a few compute workloads really need it
 590          * (the register is whitelisted in hardware now, so UMDs can opt in
 591          * for coherency if they have a good reason).
 592          */
 593         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 594
 595         /* Wa_2006611047:icl (pre-prod)
 596          * Formerly known as WaDisableImprovedTdlClkGating
 597          */
 598         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 599                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 600                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 601
 602         /* Wa_2006665173:icl (pre-prod) */
 603         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 604                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 605                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 606
 607         /* WaEnableFloatBlendOptimization:icl */
 608         wa_write_masked_or(wal,
 609                            GEN10_CACHE_MODE_SS,
 610                            0, /* write-only, so skip validation */
 611                            _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 612
 613         /* WaDisableGPGPUMidThreadPreemption:icl */
 614         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 615                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 616                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 617
 618         /* allow headerless messages for preemptible GPGPU context */
 619         WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
 620                           GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 621
 622         /* Wa_1604278689:icl,ehl */
 623         wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
 624         wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER,
 625                            0, /* write-only register; skip validation */
 626                            0xFFFFFFFF);
 627
 628         /* Wa_1406306137:icl,ehl */
 629         wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
 630 }
 631
 632 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
 633                                        struct i915_wa_list *wal)
 634 {
 635         /*
 636          * Wa_1409142259:tgl
 637          * Wa_1409347922:tgl
 638          * Wa_1409252684:tgl
 639          * Wa_1409217633:tgl
 640          * Wa_1409207793:tgl
 641          * Wa_1409178076:tgl
 642          * Wa_1408979724:tgl
 643          * Wa_14010443199:rkl
 644          * Wa_14010698770:rkl
 645          */
 646         WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 647                           GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 648
 649         /* WaDisableGPGPUMidThreadPreemption:gen12 */
 650         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 651                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 652                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 653 }
 654
 655 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
 656                                      struct i915_wa_list *wal)
 657 {
 658         gen12_ctx_workarounds_init(engine, wal);
 659
 660         /*
 661          * Wa_1604555607:tgl,rkl
 662          *
 663          * Note that the implementation of this workaround is further modified
 664          * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
 665          * FF_MODE2 register will return the wrong value when read. The default
 666          * value for this register is zero for all fields and there are no bit
 667          * masks. So instead of doing a RMW we should just write the GS Timer
 668          * and TDS timer values for Wa_1604555607 and Wa_16011163337.
 669          */
 670         wa_add(wal,
 671                FF_MODE2,
 672                FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK,
 673                FF_MODE2_GS_TIMER_224  | FF_MODE2_TDS_TIMER_128,
 674                0);
 675 }
 676
 677 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
 678                                      struct i915_wa_list *wal)
 679 {
 680         gen12_ctx_workarounds_init(engine, wal);
 681
 682         /* Wa_1409044764 */
 683         WA_CLR_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 684                           DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
 685
 686         /* Wa_22010493298 */
 687         WA_SET_BIT_MASKED(HIZ_CHICKEN,
 688                           DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
 689 }
 690
 691 static void
 692 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 693                            struct i915_wa_list *wal,
 694                            const char *name)
 695 {
 696         struct drm_i915_private *i915 = engine->i915;
 697
 698         if (engine->class != RENDER_CLASS)
 699                 return;
 700
 701         wa_init_start(wal, name, engine->name);
 702
 703         if (IS_DG1(i915))
 704                 dg1_ctx_workarounds_init(engine, wal);
 705         else if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915))
 706                 tgl_ctx_workarounds_init(engine, wal);
 707         else if (IS_GEN(i915, 12))
 708                 gen12_ctx_workarounds_init(engine, wal);
 709         else if (IS_GEN(i915, 11))
 710                 icl_ctx_workarounds_init(engine, wal);
 711         else if (IS_CANNONLAKE(i915))
 712                 cnl_ctx_workarounds_init(engine, wal);
 713         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 714                 cfl_ctx_workarounds_init(engine, wal);
 715         else if (IS_GEMINILAKE(i915))
 716                 glk_ctx_workarounds_init(engine, wal);
 717         else if (IS_KABYLAKE(i915))
 718                 kbl_ctx_workarounds_init(engine, wal);
 719         else if (IS_BROXTON(i915))
 720                 bxt_ctx_workarounds_init(engine, wal);
 721         else if (IS_SKYLAKE(i915))
 722                 skl_ctx_workarounds_init(engine, wal);
 723         else if (IS_CHERRYVIEW(i915))
 724                 chv_ctx_workarounds_init(engine, wal);
 725         else if (IS_BROADWELL(i915))
 726                 bdw_ctx_workarounds_init(engine, wal);
 727         else if (IS_GEN(i915, 7))
 728                 gen7_ctx_workarounds_init(engine, wal);
 729         else if (IS_GEN(i915, 6))
 730                 gen6_ctx_workarounds_init(engine, wal);
 731         else if (INTEL_GEN(i915) < 8)
 732                 return;
 733         else
 734                 MISSING_CASE(INTEL_GEN(i915));
 735
 736         wa_init_finish(wal);
 737 }
 738
 739 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 740 {
 741         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 742 }
 743
 744 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 745 {
 746         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 747         struct i915_wa *wa;
 748         unsigned int i;
 749         u32 *cs;
 750         int ret;
 751
 752         if (wal->count == 0)
 753                 return 0;
 754
 755         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 756         if (ret)
 757                 return ret;
 758
 759         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 760         if (IS_ERR(cs))
 761                 return PTR_ERR(cs);
 762
 763         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 764         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 765                 *cs++ = i915_mmio_reg_offset(wa->reg);
 766                 *cs++ = wa->set;
 767         }
 768         *cs++ = MI_NOOP;
 769
 770         intel_ring_advance(rq, cs);
 771
 772         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 773         if (ret)
 774                 return ret;
 775
 776         return 0;
 777 }
 778
 779 static void
 780 gen4_gt_workarounds_init(struct drm_i915_private *i915,
 781                          struct i915_wa_list *wal)
 782 {
 783         /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
 784         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 785 }
 786
 787 static void
 788 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 789 {
 790         gen4_gt_workarounds_init(i915, wal);
 791
 792         /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
 793         wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
 794 }
 795
 796 static void
 797 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 798 {
 799         g4x_gt_workarounds_init(i915, wal);
 800
 801         wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
 802 }
 803
 804 static void
 805 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 806 {
 807         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
 808         wa_masked_en(wal,
 809                      _3D_CHICKEN,
 810                      _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
 811
 812         /* WaDisable_RenderCache_OperationalFlush:snb */
 813         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 814
 815         /*
 816          * BSpec recommends 8x4 when MSAA is used,
 817          * however in practice 16x4 seems fastest.
 818          *
 819          * Note that PS/WM thread counts depend on the WIZ hashing
 820          * disable bit, which we don't touch here, but it's good
 821          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 822          */
 823         wa_add(wal,
 824                GEN6_GT_MODE, 0,
 825                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 826                GEN6_WIZ_HASHING_16x4);
 827
 828         wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB);
 829
 830         wa_masked_en(wal,
 831                      _3D_CHICKEN3,
 832                      /* WaStripsFansDisableFastClipPerformanceFix:snb */
 833                      _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
 834                      /*
 835                       * Bspec says:
 836                       * "This bit must be set if 3DSTATE_CLIP clip mode is set
 837                       * to normal and 3DSTATE_SF number of SF output attributes
 838                       * is more than 16."
 839                       */
 840                    _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
 841 }
 842
 843 static void
 844 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 845 {
 846         /* WaDisableEarlyCull:ivb */
 847         wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
 848
 849         /* WaDisablePSDDualDispatchEnable:ivb */
 850         if (IS_IVB_GT1(i915))
 851                 wa_masked_en(wal,
 852                              GEN7_HALF_SLICE_CHICKEN1,
 853                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
 854
 855         /* WaDisable_RenderCache_OperationalFlush:ivb */
 856         wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
 857
 858         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 859         wa_masked_dis(wal,
 860                       GEN7_COMMON_SLICE_CHICKEN1,
 861                       GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 862
 863         /* WaApplyL3ControlAndL3ChickenMode:ivb */
 864         wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
 865         wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 866
 867         /* WaForceL3Serialization:ivb */
 868         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 869
 870         /*
 871          * WaVSThreadDispatchOverride:ivb,vlv
 872          *
 873          * This actually overrides the dispatch
 874          * mode for all thread types.
 875          */
 876         wa_write_masked_or(wal, GEN7_FF_THREAD_MODE,
 877                            GEN7_FF_SCHED_MASK,
 878                            GEN7_FF_TS_SCHED_HW |
 879                            GEN7_FF_VS_SCHED_HW |
 880                            GEN7_FF_DS_SCHED_HW);
 881
 882         if (0) { /* causes HiZ corruption on ivb:gt1 */
 883                 /* enable HiZ Raw Stall Optimization */
 884                 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 885         }
 886
 887         /* WaDisable4x2SubspanOptimization:ivb */
 888         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 889
 890         /*
 891          * BSpec recommends 8x4 when MSAA is used,
 892          * however in practice 16x4 seems fastest.
 893          *
 894          * Note that PS/WM thread counts depend on the WIZ hashing
 895          * disable bit, which we don't touch here, but it's good
 896          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 897          */
 898         wa_add(wal, GEN7_GT_MODE, 0,
 899                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 900                GEN6_WIZ_HASHING_16x4);
 901 }
 902
 903 static void
 904 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 905 {
 906         /* WaDisableEarlyCull:vlv */
 907         wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
 908
 909         /* WaPsdDispatchEnable:vlv */
 910         /* WaDisablePSDDualDispatchEnable:vlv */
 911         wa_masked_en(wal,
 912                      GEN7_HALF_SLICE_CHICKEN1,
 913                      GEN7_MAX_PS_THREAD_DEP |
 914                      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
 915
 916         /* WaDisable_RenderCache_OperationalFlush:vlv */
 917         wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
 918
 919         /* WaForceL3Serialization:vlv */
 920         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 921
 922         /*
 923          * WaVSThreadDispatchOverride:ivb,vlv
 924          *
 925          * This actually overrides the dispatch
 926          * mode for all thread types.
 927          */
 928         wa_write_masked_or(wal,
 929                            GEN7_FF_THREAD_MODE,
 930                            GEN7_FF_SCHED_MASK,
 931                            GEN7_FF_TS_SCHED_HW |
 932                            GEN7_FF_VS_SCHED_HW |
 933                            GEN7_FF_DS_SCHED_HW);
 934
 935         /*
 936          * BSpec says this must be set, even though
 937          * WaDisable4x2SubspanOptimization isn't listed for VLV.
 938          */
 939         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 940
 941         /*
 942          * BSpec recommends 8x4 when MSAA is used,
 943          * however in practice 16x4 seems fastest.
 944          *
 945          * Note that PS/WM thread counts depend on the WIZ hashing
 946          * disable bit, which we don't touch here, but it's good
 947          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 948          */
 949         wa_add(wal, GEN7_GT_MODE, 0,
 950                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 951                GEN6_WIZ_HASHING_16x4);
 952
 953         /*
 954          * WaIncreaseL3CreditsForVLVB0:vlv
 955          * This is the hardware default actually.
 956          */
 957         wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
 958 }
 959
 960 static void
 961 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 962 {
 963         /* L3 caching of data atomics doesn't work -- disable it. */
 964         wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
 965
 966         wa_add(wal,
 967                HSW_ROW_CHICKEN3, 0,
 968                _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
 969                 0 /* XXX does this reg exist? */);
 970
 971         /* WaVSRefCountFullforceMissDisable:hsw */
 972         wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
 973
 974         wa_masked_dis(wal,
 975                       CACHE_MODE_0_GEN7,
 976                       /* WaDisable_RenderCache_OperationalFlush:hsw */
 977                       RC_OP_FLUSH_ENABLE |
 978                       /* enable HiZ Raw Stall Optimization */
 979                       HIZ_RAW_STALL_OPT_DISABLE);
 980
 981         /* WaDisable4x2SubspanOptimization:hsw */
 982         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 983
 984         /*
 985          * BSpec recommends 8x4 when MSAA is used,
 986          * however in practice 16x4 seems fastest.
 987          *
 988          * Note that PS/WM thread counts depend on the WIZ hashing
 989          * disable bit, which we don't touch here, but it's good
 990          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 991          */
 992         wa_add(wal, GEN7_GT_MODE, 0,
 993                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 994                GEN6_WIZ_HASHING_16x4);
 995
 996         /* WaSampleCChickenBitEnable:hsw */
 997         wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
 998 }
 999
1000 static void
1001 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1002 {
1003         /* WaDisableKillLogic:bxt,skl,kbl */
1004         if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
1005                 wa_write_or(wal,
1006                             GAM_ECOCHK,
1007                             ECOCHK_DIS_TLB);
1008
1009         if (HAS_LLC(i915)) {
1010                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
1011                  *
1012                  * Must match Display Engine. See
1013                  * WaCompressedResourceDisplayNewHashMode.
1014                  */
1015                 wa_write_or(wal,
1016                             MMCD_MISC_CTRL,
1017                             MMCD_PCLA | MMCD_HOTSPOT_EN);
1018         }
1019
1020         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1021         wa_write_or(wal,
1022                     GAM_ECOCHK,
1023                     BDW_DISABLE_HDC_INVALIDATION);
1024 }
1025
1026 static void
1027 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1028 {
1029         gen9_gt_workarounds_init(i915, wal);
1030
1031         /* WaDisableGafsUnitClkGating:skl */
1032         wa_write_or(wal,
1033                     GEN7_UCGCTL4,
1034                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1035
1036         /* WaInPlaceDecompressionHang:skl */
1037         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
1038                 wa_write_or(wal,
1039                             GEN9_GAMT_ECO_REG_RW_IA,
1040                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1041 }
1042
1043 static void
1044 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1045 {
1046         gen9_gt_workarounds_init(i915, wal);
1047
1048         /* WaInPlaceDecompressionHang:bxt */
1049         wa_write_or(wal,
1050                     GEN9_GAMT_ECO_REG_RW_IA,
1051                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1052 }
1053
1054 static void
1055 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1056 {
1057         gen9_gt_workarounds_init(i915, wal);
1058
1059         /* WaDisableDynamicCreditSharing:kbl */
1060         if (IS_KBL_GT_REVID(i915, 0, KBL_REVID_B0))
1061                 wa_write_or(wal,
1062                             GAMT_CHKN_BIT_REG,
1063                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1064
1065         /* WaDisableGafsUnitClkGating:kbl */
1066         wa_write_or(wal,
1067                     GEN7_UCGCTL4,
1068                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1069
1070         /* WaInPlaceDecompressionHang:kbl */
1071         wa_write_or(wal,
1072                     GEN9_GAMT_ECO_REG_RW_IA,
1073                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1074 }
1075
1076 static void
1077 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1078 {
1079         gen9_gt_workarounds_init(i915, wal);
1080 }
1081
1082 static void
1083 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1084 {
1085         gen9_gt_workarounds_init(i915, wal);
1086
1087         /* WaDisableGafsUnitClkGating:cfl */
1088         wa_write_or(wal,
1089                     GEN7_UCGCTL4,
1090                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1091
1092         /* WaInPlaceDecompressionHang:cfl */
1093         wa_write_or(wal,
1094                     GEN9_GAMT_ECO_REG_RW_IA,
1095                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1096 }
1097
1098 static void
1099 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1100 {
1101         const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
1102         unsigned int slice, subslice;
1103         u32 l3_en, mcr, mcr_mask;
1104
1105         GEM_BUG_ON(INTEL_GEN(i915) < 10);
1106
1107         /*
1108          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
1109          * L3Banks could be fused off in single slice scenario. If that is
1110          * the case, we might need to program MCR select to a valid L3Bank
1111          * by default, to make sure we correctly read certain registers
1112          * later on (in the range 0xB100 - 0xB3FF).
1113          *
1114          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
1115          * Before any MMIO read into slice/subslice specific registers, MCR
1116          * packet control register needs to be programmed to point to any
1117          * enabled s/ss pair. Otherwise, incorrect values will be returned.
1118          * This means each subsequent MMIO read will be forwarded to an
1119          * specific s/ss combination, but this is OK since these registers
1120          * are consistent across s/ss in almost all cases. In the rare
1121          * occasions, such as INSTDONE, where this value is dependent
1122          * on s/ss combo, the read should be done with read_subslice_reg.
1123          *
1124          * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
1125          * to which subslice, or to which L3 bank, the respective mmio reads
1126          * will go, we have to find a common index which works for both
1127          * accesses.
1128          *
1129          * Case where we cannot find a common index fortunately should not
1130          * happen in production hardware, so we only emit a warning instead of
1131          * implementing something more complex that requires checking the range
1132          * of every MMIO read.
1133          */
1134
1135         if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
1136                 u32 l3_fuse =
1137                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
1138                         GEN10_L3BANK_MASK;
1139
1140                 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
1141                 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
1142         } else {
1143                 l3_en = ~0;
1144         }
1145
1146         slice = fls(sseu->slice_mask) - 1;
1147         subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
1148         if (!subslice) {
1149                 drm_warn(&i915->drm,
1150                          "No common index found between subslice mask %x and L3 bank mask %x!\n",
1151                          intel_sseu_get_subslices(sseu, slice), l3_en);
1152                 subslice = fls(l3_en);
1153                 drm_WARN_ON(&i915->drm, !subslice);
1154         }
1155         subslice--;
1156
1157         if (INTEL_GEN(i915) >= 11) {
1158                 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1159                 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1160         } else {
1161                 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1162                 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1163         }
1164
1165         drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1166
1167         wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1168 }
1169
1170 static void
1171 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1172 {
1173         wa_init_mcr(i915, wal);
1174
1175         /* WaInPlaceDecompressionHang:cnl */
1176         wa_write_or(wal,
1177                     GEN9_GAMT_ECO_REG_RW_IA,
1178                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1179 }
1180
1181 static void
1182 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1183 {
1184         wa_init_mcr(i915, wal);
1185
1186         /* WaInPlaceDecompressionHang:icl */
1187         wa_write_or(wal,
1188                     GEN9_GAMT_ECO_REG_RW_IA,
1189                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1190
1191         /* WaModifyGamTlbPartitioning:icl */
1192         wa_write_masked_or(wal,
1193                            GEN11_GACB_PERF_CTRL,
1194                            GEN11_HASH_CTRL_MASK,
1195                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1196
1197         /* Wa_1405766107:icl
1198          * Formerly known as WaCL2SFHalfMaxAlloc
1199          */
1200         wa_write_or(wal,
1201                     GEN11_LSN_UNSLCVC,
1202                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1203                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1204
1205         /* Wa_220166154:icl
1206          * Formerly known as WaDisCtxReload
1207          */
1208         wa_write_or(wal,
1209                     GEN8_GAMW_ECO_DEV_RW_IA,
1210                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1211
1212         /* Wa_1405779004:icl (pre-prod) */
1213         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1214                 wa_write_or(wal,
1215                             SLICE_UNIT_LEVEL_CLKGATE,
1216                             MSCUNIT_CLKGATE_DIS);
1217
1218         /* Wa_1406838659:icl (pre-prod) */
1219         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1220                 wa_write_or(wal,
1221                             INF_UNIT_LEVEL_CLKGATE,
1222                             CGPSF_CLKGATE_DIS);
1223
1224         /* Wa_1406463099:icl
1225          * Formerly known as WaGamTlbPendError
1226          */
1227         wa_write_or(wal,
1228                     GAMT_CHKN_BIT_REG,
1229                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
1230
1231         /* Wa_1607087056:icl,ehl,jsl */
1232         if (IS_ICELAKE(i915) ||
1233                 IS_JSL_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) {
1234                 wa_write_or(wal,
1235                             SLICE_UNIT_LEVEL_CLKGATE,
1236                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1237         }
1238 }
1239
1240 static void
1241 gen12_gt_workarounds_init(struct drm_i915_private *i915,
1242                           struct i915_wa_list *wal)
1243 {
1244         wa_init_mcr(i915, wal);
1245 }
1246
1247 static void
1248 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1249 {
1250         gen12_gt_workarounds_init(i915, wal);
1251
1252         /* Wa_1409420604:tgl */
1253         if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1254                 wa_write_or(wal,
1255                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1256                             CPSSUNIT_CLKGATE_DIS);
1257
1258         /* Wa_1607087056:tgl also know as BUG:1409180338 */
1259         if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1260                 wa_write_or(wal,
1261                             SLICE_UNIT_LEVEL_CLKGATE,
1262                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1263 }
1264
1265 static void
1266 dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1267 {
1268         gen12_gt_workarounds_init(i915, wal);
1269
1270         /* Wa_1607087056:dg1 */
1271         if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0))
1272                 wa_write_or(wal,
1273                             SLICE_UNIT_LEVEL_CLKGATE,
1274                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1275
1276         /* Wa_1409420604:dg1 */
1277         if (IS_DG1(i915))
1278                 wa_write_or(wal,
1279                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1280                             CPSSUNIT_CLKGATE_DIS);
1281
1282         /* Wa_1408615072:dg1 */
1283         /* Empirical testing shows this register is unaffected by engine reset. */
1284         if (IS_DG1(i915))
1285                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1286                             VSUNIT_CLKGATE_DIS_TGL);
1287 }
1288
1289 static void
1290 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1291 {
1292         if (IS_DG1(i915))
1293                 dg1_gt_workarounds_init(i915, wal);
1294         else if (IS_TIGERLAKE(i915))
1295                 tgl_gt_workarounds_init(i915, wal);
1296         else if (IS_GEN(i915, 12))
1297                 gen12_gt_workarounds_init(i915, wal);
1298         else if (IS_GEN(i915, 11))
1299                 icl_gt_workarounds_init(i915, wal);
1300         else if (IS_CANNONLAKE(i915))
1301                 cnl_gt_workarounds_init(i915, wal);
1302         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1303                 cfl_gt_workarounds_init(i915, wal);
1304         else if (IS_GEMINILAKE(i915))
1305                 glk_gt_workarounds_init(i915, wal);
1306         else if (IS_KABYLAKE(i915))
1307                 kbl_gt_workarounds_init(i915, wal);
1308         else if (IS_BROXTON(i915))
1309                 bxt_gt_workarounds_init(i915, wal);
1310         else if (IS_SKYLAKE(i915))
1311                 skl_gt_workarounds_init(i915, wal);
1312         else if (IS_HASWELL(i915))
1313                 hsw_gt_workarounds_init(i915, wal);
1314         else if (IS_VALLEYVIEW(i915))
1315                 vlv_gt_workarounds_init(i915, wal);
1316         else if (IS_IVYBRIDGE(i915))
1317                 ivb_gt_workarounds_init(i915, wal);
1318         else if (IS_GEN(i915, 6))
1319                 snb_gt_workarounds_init(i915, wal);
1320         else if (IS_GEN(i915, 5))
1321                 ilk_gt_workarounds_init(i915, wal);
1322         else if (IS_G4X(i915))
1323                 g4x_gt_workarounds_init(i915, wal);
1324         else if (IS_GEN(i915, 4))
1325                 gen4_gt_workarounds_init(i915, wal);
1326         else if (INTEL_GEN(i915) <= 8)
1327                 return;
1328         else
1329                 MISSING_CASE(INTEL_GEN(i915));
1330 }
1331
1332 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1333 {
1334         struct i915_wa_list *wal = &i915->gt_wa_list;
1335
1336         wa_init_start(wal, "GT", "global");
1337         gt_init_workarounds(i915, wal);
1338         wa_init_finish(wal);
1339 }
1340
1341 static enum forcewake_domains
1342 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1343 {
1344         enum forcewake_domains fw = 0;
1345         struct i915_wa *wa;
1346         unsigned int i;
1347
1348         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1349                 fw |= intel_uncore_forcewake_for_reg(uncore,
1350                                                      wa->reg,
1351                                                      FW_REG_READ |
1352                                                      FW_REG_WRITE);
1353
1354         return fw;
1355 }
1356
1357 static bool
1358 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1359 {
1360         if ((cur ^ wa->set) & wa->read) {
1361                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
1362                           name, from, i915_mmio_reg_offset(wa->reg),
1363                           cur, cur & wa->read, wa->set);
1364
1365                 return false;
1366         }
1367
1368         return true;
1369 }
1370
1371 static void
1372 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1373 {
1374         enum forcewake_domains fw;
1375         unsigned long flags;
1376         struct i915_wa *wa;
1377         unsigned int i;
1378
1379         if (!wal->count)
1380                 return;
1381
1382         fw = wal_get_fw_for_rmw(uncore, wal);
1383
1384         spin_lock_irqsave(&uncore->lock, flags);
1385         intel_uncore_forcewake_get__locked(uncore, fw);
1386
1387         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1388                 if (wa->clr)
1389                         intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1390                 else
1391                         intel_uncore_write_fw(uncore, wa->reg, wa->set);
1392                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1393                         wa_verify(wa,
1394                                   intel_uncore_read_fw(uncore, wa->reg),
1395                                   wal->name, "application");
1396         }
1397
1398         intel_uncore_forcewake_put__locked(uncore, fw);
1399         spin_unlock_irqrestore(&uncore->lock, flags);
1400 }
1401
1402 void intel_gt_apply_workarounds(struct intel_gt *gt)
1403 {
1404         wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1405 }
1406
1407 static bool wa_list_verify(struct intel_uncore *uncore,
1408                            const struct i915_wa_list *wal,
1409                            const char *from)
1410 {
1411         struct i915_wa *wa;
1412         unsigned int i;
1413         bool ok = true;
1414
1415         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1416                 ok &= wa_verify(wa,
1417                                 intel_uncore_read(uncore, wa->reg),
1418                                 wal->name, from);
1419
1420         return ok;
1421 }
1422
1423 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1424 {
1425         return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1426 }
1427
1428 static inline bool is_nonpriv_flags_valid(u32 flags)
1429 {
1430         /* Check only valid flag bits are set */
1431         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1432                 return false;
1433
1434         /* NB: Only 3 out of 4 enum values are valid for access field */
1435         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1436             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1437                 return false;
1438
1439         return true;
1440 }
1441
1442 static void
1443 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1444 {
1445         struct i915_wa wa = {
1446                 .reg = reg
1447         };
1448
1449         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1450                 return;
1451
1452         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1453                 return;
1454
1455         wa.reg.reg |= flags;
1456         _wa_add(wal, &wa);
1457 }
1458
1459 static void
1460 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1461 {
1462         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1463 }
1464
1465 static void gen9_whitelist_build(struct i915_wa_list *w)
1466 {
1467         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1468         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1469
1470         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1471         whitelist_reg(w, GEN8_CS_CHICKEN1);
1472
1473         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1474         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1475
1476         /* WaSendPushConstantsFromMMIO:skl,bxt */
1477         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1478 }
1479
1480 static void skl_whitelist_build(struct intel_engine_cs *engine)
1481 {
1482         struct i915_wa_list *w = &engine->whitelist;
1483
1484         if (engine->class != RENDER_CLASS)
1485                 return;
1486
1487         gen9_whitelist_build(w);
1488
1489         /* WaDisableLSQCROPERFforOCL:skl */
1490         whitelist_reg(w, GEN8_L3SQCREG4);
1491 }
1492
1493 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1494 {
1495         if (engine->class != RENDER_CLASS)
1496                 return;
1497
1498         gen9_whitelist_build(&engine->whitelist);
1499 }
1500
1501 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1502 {
1503         struct i915_wa_list *w = &engine->whitelist;
1504
1505         if (engine->class != RENDER_CLASS)
1506                 return;
1507
1508         gen9_whitelist_build(w);
1509
1510         /* WaDisableLSQCROPERFforOCL:kbl */
1511         whitelist_reg(w, GEN8_L3SQCREG4);
1512 }
1513
1514 static void glk_whitelist_build(struct intel_engine_cs *engine)
1515 {
1516         struct i915_wa_list *w = &engine->whitelist;
1517
1518         if (engine->class != RENDER_CLASS)
1519                 return;
1520
1521         gen9_whitelist_build(w);
1522
1523         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1524         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1525 }
1526
1527 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1528 {
1529         struct i915_wa_list *w = &engine->whitelist;
1530
1531         if (engine->class != RENDER_CLASS)
1532                 return;
1533
1534         gen9_whitelist_build(w);
1535
1536         /*
1537          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1538          *
1539          * This covers 4 register which are next to one another :
1540          *   - PS_INVOCATION_COUNT
1541          *   - PS_INVOCATION_COUNT_UDW
1542          *   - PS_DEPTH_COUNT
1543          *   - PS_DEPTH_COUNT_UDW
1544          */
1545         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1546                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1547                           RING_FORCE_TO_NONPRIV_RANGE_4);
1548 }
1549
1550 static void cml_whitelist_build(struct intel_engine_cs *engine)
1551 {
1552         struct i915_wa_list *w = &engine->whitelist;
1553
1554         if (engine->class != RENDER_CLASS)
1555                 whitelist_reg_ext(w,
1556                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1557                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1558
1559         cfl_whitelist_build(engine);
1560 }
1561
1562 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1563 {
1564         struct i915_wa_list *w = &engine->whitelist;
1565
1566         if (engine->class != RENDER_CLASS)
1567                 return;
1568
1569         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1570         whitelist_reg(w, GEN8_CS_CHICKEN1);
1571 }
1572
1573 static void icl_whitelist_build(struct intel_engine_cs *engine)
1574 {
1575         struct i915_wa_list *w = &engine->whitelist;
1576
1577         switch (engine->class) {
1578         case RENDER_CLASS:
1579                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1580                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1581
1582                 /* WaAllowUMDToModifySamplerMode:icl */
1583                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1584
1585                 /* WaEnableStateCacheRedirectToCS:icl */
1586                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1587
1588                 /*
1589                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1590                  *
1591                  * This covers 4 register which are next to one another :
1592                  *   - PS_INVOCATION_COUNT
1593                  *   - PS_INVOCATION_COUNT_UDW
1594                  *   - PS_DEPTH_COUNT
1595                  *   - PS_DEPTH_COUNT_UDW
1596                  */
1597                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1598                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1599                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1600                 break;
1601
1602         case VIDEO_DECODE_CLASS:
1603                 /* hucStatusRegOffset */
1604                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1605                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1606                 /* hucUKernelHdrInfoRegOffset */
1607                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1608                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1609                 /* hucStatus2RegOffset */
1610                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1611                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1612                 whitelist_reg_ext(w,
1613                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1614                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1615                 break;
1616
1617         default:
1618                 whitelist_reg_ext(w,
1619                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1620                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1621                 break;
1622         }
1623 }
1624
1625 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1626 {
1627         struct i915_wa_list *w = &engine->whitelist;
1628
1629         switch (engine->class) {
1630         case RENDER_CLASS:
1631                 /*
1632                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1633                  * Wa_1408556865:tgl
1634                  *
1635                  * This covers 4 registers which are next to one another :
1636                  *   - PS_INVOCATION_COUNT
1637                  *   - PS_INVOCATION_COUNT_UDW
1638                  *   - PS_DEPTH_COUNT
1639                  *   - PS_DEPTH_COUNT_UDW
1640                  */
1641                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1642                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1643                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1644
1645                 /* Wa_1808121037:tgl */
1646                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1647
1648                 /* Wa_1806527549:tgl */
1649                 whitelist_reg(w, HIZ_CHICKEN);
1650                 break;
1651         default:
1652                 whitelist_reg_ext(w,
1653                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1654                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1655                 break;
1656         }
1657 }
1658
1659 static void dg1_whitelist_build(struct intel_engine_cs *engine)
1660 {
1661         struct i915_wa_list *w = &engine->whitelist;
1662
1663         tgl_whitelist_build(engine);
1664
1665         /* GEN:BUG:1409280441:dg1 */
1666         if (IS_DG1_REVID(engine->i915, DG1_REVID_A0, DG1_REVID_A0) &&
1667             (engine->class == RENDER_CLASS ||
1668              engine->class == COPY_ENGINE_CLASS))
1669                 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1670                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1671 }
1672
1673 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1674 {
1675         struct drm_i915_private *i915 = engine->i915;
1676         struct i915_wa_list *w = &engine->whitelist;
1677
1678         wa_init_start(w, "whitelist", engine->name);
1679
1680         if (IS_DG1(i915))
1681                 dg1_whitelist_build(engine);
1682         else if (IS_GEN(i915, 12))
1683                 tgl_whitelist_build(engine);
1684         else if (IS_GEN(i915, 11))
1685                 icl_whitelist_build(engine);
1686         else if (IS_CANNONLAKE(i915))
1687                 cnl_whitelist_build(engine);
1688         else if (IS_COMETLAKE(i915))
1689                 cml_whitelist_build(engine);
1690         else if (IS_COFFEELAKE(i915))
1691                 cfl_whitelist_build(engine);
1692         else if (IS_GEMINILAKE(i915))
1693                 glk_whitelist_build(engine);
1694         else if (IS_KABYLAKE(i915))
1695                 kbl_whitelist_build(engine);
1696         else if (IS_BROXTON(i915))
1697                 bxt_whitelist_build(engine);
1698         else if (IS_SKYLAKE(i915))
1699                 skl_whitelist_build(engine);
1700         else if (INTEL_GEN(i915) <= 8)
1701                 return;
1702         else
1703                 MISSING_CASE(INTEL_GEN(i915));
1704
1705         wa_init_finish(w);
1706 }
1707
1708 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1709 {
1710         const struct i915_wa_list *wal = &engine->whitelist;
1711         struct intel_uncore *uncore = engine->uncore;
1712         const u32 base = engine->mmio_base;
1713         struct i915_wa *wa;
1714         unsigned int i;
1715
1716         if (!wal->count)
1717                 return;
1718
1719         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1720                 intel_uncore_write(uncore,
1721                                    RING_FORCE_TO_NONPRIV(base, i),
1722                                    i915_mmio_reg_offset(wa->reg));
1723
1724         /* And clear the rest just in case of garbage */
1725         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1726                 intel_uncore_write(uncore,
1727                                    RING_FORCE_TO_NONPRIV(base, i),
1728                                    i915_mmio_reg_offset(RING_NOPID(base)));
1729 }
1730
1731 static void
1732 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1733 {
1734         struct drm_i915_private *i915 = engine->i915;
1735
1736         if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1737             IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1738                 /*
1739                  * Wa_1607138336:tgl[a0],dg1[a0]
1740                  * Wa_1607063988:tgl[a0],dg1[a0]
1741                  */
1742                 wa_write_or(wal,
1743                             GEN9_CTX_PREEMPT_REG,
1744                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1745         }
1746
1747         if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1748                 /*
1749                  * Wa_1606679103:tgl
1750                  * (see also Wa_1606682166:icl)
1751                  */
1752                 wa_write_or(wal,
1753                             GEN7_SARCHKMD,
1754                             GEN7_DISABLE_SAMPLER_PREFETCH);
1755
1756                 /* Wa_1408615072:tgl */
1757                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1758                             VSUNIT_CLKGATE_DIS_TGL);
1759         }
1760
1761         if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1762                 /* Wa_1606931601:tgl,rkl,dg1 */
1763                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1764
1765                 /*
1766                  * Wa_1407928979:tgl A*
1767                  * Wa_18011464164:tgl[B0+],dg1[B0+]
1768                  * Wa_22010931296:tgl[B0+],dg1[B0+]
1769                  * Wa_14010919138:rkl, dg1
1770                  */
1771                 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1772                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1773         }
1774
1775         if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1776             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1777                 /* Wa_1409804808:tgl,rkl,dg1[a0] */
1778                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1779                              GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1780
1781                 /*
1782                  * Wa_1409085225:tgl
1783                  * Wa_14010229206:tgl,rkl,dg1[a0]
1784                  */
1785                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1786
1787                 /*
1788                  * Wa_1607030317:tgl
1789                  * Wa_1607186500:tgl
1790                  * Wa_1607297627:tgl,rkl,dg1[a0]
1791                  *
1792                  * On TGL and RKL there are multiple entries for this WA in the
1793                  * BSpec; some indicate this is an A0-only WA, others indicate
1794                  * it applies to all steppings so we trust the "all steppings."
1795                  * For DG1 this only applies to A0.
1796                  */
1797                 wa_masked_en(wal,
1798                              GEN6_RC_SLEEP_PSMI_CONTROL,
1799                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1800                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1801
1802                 /*
1803                  * Wa_1606700617:tgl
1804                  * Wa_22010271021:tgl,rkl
1805                  */
1806                 wa_masked_en(wal,
1807                              GEN9_CS_DEBUG_MODE1,
1808                              FF_DOP_CLOCK_GATE_DISABLE);
1809         }
1810
1811         if (IS_GEN(i915, 12)) {
1812                 /* Wa_1406941453:gen12 */
1813                 wa_masked_en(wal,
1814                              GEN10_SAMPLER_MODE,
1815                              ENABLE_SMALLPL);
1816         }
1817
1818         if (IS_GEN(i915, 11)) {
1819                 /* This is not an Wa. Enable for better image quality */
1820                 wa_masked_en(wal,
1821                              _3D_CHICKEN3,
1822                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1823
1824                 /* WaPipelineFlushCoherentLines:icl */
1825                 wa_write_or(wal,
1826                             GEN8_L3SQCREG4,
1827                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1828
1829                 /*
1830                  * Wa_1405543622:icl
1831                  * Formerly known as WaGAPZPriorityScheme
1832                  */
1833                 wa_write_or(wal,
1834                             GEN8_GARBCNTL,
1835                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1836
1837                 /*
1838                  * Wa_1604223664:icl
1839                  * Formerly known as WaL3BankAddressHashing
1840                  */
1841                 wa_write_masked_or(wal,
1842                                    GEN8_GARBCNTL,
1843                                    GEN11_HASH_CTRL_EXCL_MASK,
1844                                    GEN11_HASH_CTRL_EXCL_BIT0);
1845                 wa_write_masked_or(wal,
1846                                    GEN11_GLBLINVL,
1847                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1848                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1849
1850                 /*
1851                  * Wa_1405733216:icl
1852                  * Formerly known as WaDisableCleanEvicts
1853                  */
1854                 wa_write_or(wal,
1855                             GEN8_L3SQCREG4,
1856                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1857
1858                 /* WaForwardProgressSoftReset:icl */
1859                 wa_write_or(wal,
1860                             GEN10_SCRATCH_LNCF2,
1861                             PMFLUSHDONE_LNICRSDROP |
1862                             PMFLUSH_GAPL3UNBLOCK |
1863                             PMFLUSHDONE_LNEBLK);
1864
1865                 /* Wa_1406609255:icl (pre-prod) */
1866                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1867                         wa_write_or(wal,
1868                                     GEN7_SARCHKMD,
1869                                     GEN7_DISABLE_DEMAND_PREFETCH);
1870
1871                 /* Wa_1606682166:icl */
1872                 wa_write_or(wal,
1873                             GEN7_SARCHKMD,
1874                             GEN7_DISABLE_SAMPLER_PREFETCH);
1875
1876                 /* Wa_1409178092:icl */
1877                 wa_write_masked_or(wal,
1878                                    GEN11_SCRATCH2,
1879                                    GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1880                                    0);
1881
1882                 /* WaEnable32PlaneMode:icl */
1883                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1884                              GEN11_ENABLE_32_PLANE_MODE);
1885
1886                 /*
1887                  * Wa_1408615072:icl,ehl  (vsunit)
1888                  * Wa_1407596294:icl,ehl  (hsunit)
1889                  */
1890                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1891                             VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1892
1893                 /* Wa_1407352427:icl,ehl */
1894                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1895                             PSDUNIT_CLKGATE_DIS);
1896
1897                 /* Wa_1406680159:icl,ehl */
1898                 wa_write_or(wal,
1899                             SUBSLICE_UNIT_LEVEL_CLKGATE,
1900                             GWUNIT_CLKGATE_DIS);
1901
1902                 /*
1903                  * Wa_1408767742:icl[a2..forever],ehl[all]
1904                  * Wa_1605460711:icl[a0..c0]
1905                  */
1906                 wa_write_or(wal,
1907                             GEN7_FF_THREAD_MODE,
1908                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1909
1910                 /* Wa_22010271021:ehl */
1911                 if (IS_JSL_EHL(i915))
1912                         wa_masked_en(wal,
1913                                      GEN9_CS_DEBUG_MODE1,
1914                                      FF_DOP_CLOCK_GATE_DISABLE);
1915         }
1916
1917         if (IS_GEN_RANGE(i915, 9, 12)) {
1918                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1919                 wa_masked_en(wal,
1920                              GEN7_FF_SLICE_CS_CHICKEN1,
1921                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1922         }
1923
1924         if (IS_SKYLAKE(i915) ||
1925             IS_KABYLAKE(i915) ||
1926             IS_COFFEELAKE(i915) ||
1927             IS_COMETLAKE(i915)) {
1928                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1929                 wa_write_or(wal,
1930                             GEN8_GARBCNTL,
1931                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1932         }
1933
1934         if (IS_BROXTON(i915)) {
1935                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1936                 wa_masked_en(wal,
1937                              FF_SLICE_CS_CHICKEN2,
1938                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1939         }
1940
1941         if (IS_GEN(i915, 9)) {
1942                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1943                 wa_masked_en(wal,
1944                              GEN9_CSFE_CHICKEN1_RCS,
1945                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1946
1947                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1948                 wa_write_or(wal,
1949                             BDW_SCRATCH1,
1950                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1951
1952                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1953                 if (IS_GEN9_LP(i915))
1954                         wa_write_masked_or(wal,
1955                                            GEN8_L3SQCREG1,
1956                                            L3_PRIO_CREDITS_MASK,
1957                                            L3_GENERAL_PRIO_CREDITS(62) |
1958                                            L3_HIGH_PRIO_CREDITS(2));
1959
1960                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1961                 wa_write_or(wal,
1962                             GEN8_L3SQCREG4,
1963                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1964         }
1965
1966         if (IS_GEN(i915, 7))
1967                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1968                 wa_masked_en(wal,
1969                              GFX_MODE_GEN7,
1970                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1971
1972         if (IS_GEN_RANGE(i915, 6, 7))
1973                 /*
1974                  * We need to disable the AsyncFlip performance optimisations in
1975                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
1976                  * already be programmed to '1' on all products.
1977                  *
1978                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1979                  */
1980                 wa_masked_en(wal,
1981                              MI_MODE,
1982                              ASYNC_FLIP_PERF_DISABLE);
1983
1984         if (IS_GEN(i915, 6)) {
1985                 /*
1986                  * Required for the hardware to program scanline values for
1987                  * waiting
1988                  * WaEnableFlushTlbInvalidationMode:snb
1989                  */
1990                 wa_masked_en(wal,
1991                              GFX_MODE,
1992                              GFX_TLB_INVALIDATE_EXPLICIT);
1993
1994                 /*
1995                  * From the Sandybridge PRM, volume 1 part 3, page 24:
1996                  * "If this bit is set, STCunit will have LRA as replacement
1997                  *  policy. [...] This bit must be reset. LRA replacement
1998                  *  policy is not supported."
1999                  */
2000                 wa_masked_dis(wal,
2001                               CACHE_MODE_0,
2002                               CM0_STC_EVICT_DISABLE_LRA_SNB);
2003         }
2004
2005         if (IS_GEN_RANGE(i915, 4, 6))
2006                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2007                 wa_add(wal, MI_MODE,
2008                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2009                        /* XXX bit doesn't stick on Broadwater */
2010                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
2011
2012         if (IS_GEN(i915, 4))
2013                 /*
2014                  * Disable CONSTANT_BUFFER before it is loaded from the context
2015                  * image. For as it is loaded, it is executed and the stored
2016                  * address may no longer be valid, leading to a GPU hang.
2017                  *
2018                  * This imposes the requirement that userspace reload their
2019                  * CONSTANT_BUFFER on every batch, fortunately a requirement
2020                  * they are already accustomed to from before contexts were
2021                  * enabled.
2022                  */
2023                 wa_add(wal, ECOSKPD,
2024                        0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2025                        0 /* XXX bit doesn't stick on Broadwater */);
2026 }
2027
2028 static void
2029 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2030 {
2031         struct drm_i915_private *i915 = engine->i915;
2032
2033         /* WaKBLVECSSemaphoreWaitPoll:kbl */
2034         if (IS_KBL_GT_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
2035                 wa_write(wal,
2036                          RING_SEMA_WAIT_POLL(engine->mmio_base),
2037                          1);
2038         }
2039 }
2040
2041 static void
2042 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2043 {
2044         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
2045                 return;
2046
2047         if (engine->class == RENDER_CLASS)
2048                 rcs_engine_wa_init(engine, wal);
2049         else
2050                 xcs_engine_wa_init(engine, wal);
2051 }
2052
2053 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2054 {
2055         struct i915_wa_list *wal = &engine->wa_list;
2056
2057         if (INTEL_GEN(engine->i915) < 4)
2058                 return;
2059
2060         wa_init_start(wal, "engine", engine->name);
2061         engine_init_workarounds(engine, wal);
2062         wa_init_finish(wal);
2063 }
2064
2065 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2066 {
2067         wa_list_apply(engine->uncore, &engine->wa_list);
2068 }
2069
2070 static struct i915_vma *
2071 create_scratch(struct i915_address_space *vm, int count)
2072 {
2073         struct drm_i915_gem_object *obj;
2074         struct i915_vma *vma;
2075         unsigned int size;
2076         int err;
2077
2078         size = round_up(count * sizeof(u32), PAGE_SIZE);
2079         obj = i915_gem_object_create_internal(vm->i915, size);
2080         if (IS_ERR(obj))
2081                 return ERR_CAST(obj);
2082
2083         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
2084
2085         vma = i915_vma_instance(obj, vm, NULL);
2086         if (IS_ERR(vma)) {
2087                 err = PTR_ERR(vma);
2088                 goto err_obj;
2089         }
2090
2091         err = i915_vma_pin(vma, 0, 0,
2092                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2093         if (err)
2094                 goto err_obj;
2095
2096         return vma;
2097
2098 err_obj:
2099         i915_gem_object_put(obj);
2100         return ERR_PTR(err);
2101 }
2102
2103 struct mcr_range {
2104         u32 start;
2105         u32 end;
2106 };
2107
2108 static const struct mcr_range mcr_ranges_gen8[] = {
2109         { .start = 0x5500, .end = 0x55ff },
2110         { .start = 0x7000, .end = 0x7fff },
2111         { .start = 0x9400, .end = 0x97ff },
2112         { .start = 0xb000, .end = 0xb3ff },
2113         { .start = 0xe000, .end = 0xe7ff },
2114         {},
2115 };
2116
2117 static const struct mcr_range mcr_ranges_gen12[] = {
2118         { .start =  0x8150, .end =  0x815f },
2119         { .start =  0x9520, .end =  0x955f },
2120         { .start =  0xb100, .end =  0xb3ff },
2121         { .start =  0xde80, .end =  0xe8ff },
2122         { .start = 0x24a00, .end = 0x24a7f },
2123         {},
2124 };
2125
2126 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2127 {
2128         const struct mcr_range *mcr_ranges;
2129         int i;
2130
2131         if (INTEL_GEN(i915) >= 12)
2132                 mcr_ranges = mcr_ranges_gen12;
2133         else if (INTEL_GEN(i915) >= 8)
2134                 mcr_ranges = mcr_ranges_gen8;
2135         else
2136                 return false;
2137
2138         /*
2139          * Registers in these ranges are affected by the MCR selector
2140          * which only controls CPU initiated MMIO. Routing does not
2141          * work for CS access so we cannot verify them on this path.
2142          */
2143         for (i = 0; mcr_ranges[i].start; i++)
2144                 if (offset >= mcr_ranges[i].start &&
2145                     offset <= mcr_ranges[i].end)
2146                         return true;
2147
2148         return false;
2149 }
2150
2151 static int
2152 wa_list_srm(struct i915_request *rq,
2153             const struct i915_wa_list *wal,
2154             struct i915_vma *vma)
2155 {
2156         struct drm_i915_private *i915 = rq->engine->i915;
2157         unsigned int i, count = 0;
2158         const struct i915_wa *wa;
2159         u32 srm, *cs;
2160
2161         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2162         if (INTEL_GEN(i915) >= 8)
2163                 srm++;
2164
2165         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2166                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2167                         count++;
2168         }
2169
2170         cs = intel_ring_begin(rq, 4 * count);
2171         if (IS_ERR(cs))
2172                 return PTR_ERR(cs);
2173
2174         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2175                 u32 offset = i915_mmio_reg_offset(wa->reg);
2176
2177                 if (mcr_range(i915, offset))
2178                         continue;
2179
2180                 *cs++ = srm;
2181                 *cs++ = offset;
2182                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2183                 *cs++ = 0;
2184         }
2185         intel_ring_advance(rq, cs);
2186
2187         return 0;
2188 }
2189
2190 static int engine_wa_list_verify(struct intel_context *ce,
2191                                  const struct i915_wa_list * const wal,
2192                                  const char *from)
2193 {
2194         const struct i915_wa *wa;
2195         struct i915_request *rq;
2196         struct i915_vma *vma;
2197         struct i915_gem_ww_ctx ww;
2198         unsigned int i;
2199         u32 *results;
2200         int err;
2201
2202         if (!wal->count)
2203                 return 0;
2204
2205         vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
2206         if (IS_ERR(vma))
2207                 return PTR_ERR(vma);
2208
2209         intel_engine_pm_get(ce->engine);
2210         i915_gem_ww_ctx_init(&ww, false);
2211 retry:
2212         err = i915_gem_object_lock(vma->obj, &ww);
2213         if (err == 0)
2214                 err = intel_context_pin_ww(ce, &ww);
2215         if (err)
2216                 goto err_pm;
2217
2218         rq = i915_request_create(ce);
2219         if (IS_ERR(rq)) {
2220                 err = PTR_ERR(rq);
2221                 goto err_unpin;
2222         }
2223
2224         err = i915_request_await_object(rq, vma->obj, true);
2225         if (err == 0)
2226                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2227         if (err == 0)
2228                 err = wa_list_srm(rq, wal, vma);
2229
2230         i915_request_get(rq);
2231         if (err)
2232                 i915_request_set_error_once(rq, err);
2233         i915_request_add(rq);
2234
2235         if (err)
2236                 goto err_rq;
2237
2238         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2239                 err = -ETIME;
2240                 goto err_rq;
2241         }
2242
2243         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2244         if (IS_ERR(results)) {
2245                 err = PTR_ERR(results);
2246                 goto err_rq;
2247         }
2248
2249         err = 0;
2250         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2251                 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2252                         continue;
2253
2254                 if (!wa_verify(wa, results[i], wal->name, from))
2255                         err = -ENXIO;
2256         }
2257
2258         i915_gem_object_unpin_map(vma->obj);
2259
2260 err_rq:
2261         i915_request_put(rq);
2262 err_unpin:
2263         intel_context_unpin(ce);
2264 err_pm:
2265         if (err == -EDEADLK) {
2266                 err = i915_gem_ww_ctx_backoff(&ww);
2267                 if (!err)
2268                         goto retry;
2269         }
2270         i915_gem_ww_ctx_fini(&ww);
2271         intel_engine_pm_put(ce->engine);
2272         i915_vma_unpin(vma);
2273         i915_vma_put(vma);
2274         return err;
2275 }
2276
2277 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2278                                     const char *from)
2279 {
2280         return engine_wa_list_verify(engine->kernel_context,
2281                                      &engine->wa_list,
2282                                      from);
2283 }
2284
2285 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2286 #include "selftest_workarounds.c"
2287 #endif