drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_engine_pm.h"
  10 #include "intel_gt.h"
  11 #include "intel_ring.h"
  12 #include "intel_workarounds.h"
  13
  14 /**
  15  * DOC: Hardware workarounds
  16  *
  17  * This file is intended as a central place to implement most [1]_ of the
  18  * required workarounds for hardware to work as originally intended. They fall
  19  * in five basic categories depending on how/when they are applied:
  20  *
  21  * - Workarounds that touch registers that are saved/restored to/from the HW
  22  *   context image. The list is emitted (via Load Register Immediate commands)
  23  *   everytime a new context is created.
  24  * - GT workarounds. The list of these WAs is applied whenever these registers
  25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  26  * - Display workarounds. The list is applied during display clock-gating
  27  *   initialization.
  28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  29  *   them directly. This is just a special case of a MMMIO workaround (as we
  30  *   write the list of these to/be-whitelisted registers to some special HW
  31  *   registers).
  32  * - Workaround batchbuffers, that get executed automatically by the hardware
  33  *   on every HW context restore.
  34  *
  35  * .. [1] Please notice that there are other WAs that, due to their nature,
  36  *    cannot be applied from a central place. Those are peppered around the rest
  37  *    of the code, as needed.
  38  *
  39  * .. [2] Technically, some registers are powercontext saved & restored, so they
  40  *    survive a suspend/resume. In practice, writing them again is not too
  41  *    costly and simplifies things. We can revisit this in the future.
  42  *
  43  * Layout
  44  * ~~~~~~
  45  *
  46  * Keep things in this file ordered by WA type, as per the above (context, GT,
  47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  48  * following order:
  49  *
  50  * - Infrastructure functions and macros
  51  * - WAs per platform in standard gen/chrono order
  52  * - Public functions to init or apply the given workaround type.
  53  */
  54
  55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  56 {
  57         wal->name = name;
  58         wal->engine_name = engine_name;
  59 }
  60
  61 #define WA_LIST_CHUNK (1 << 4)
  62
  63 static void wa_init_finish(struct i915_wa_list *wal)
  64 {
  65         /* Trim unused entries. */
  66         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  67                 struct i915_wa *list = kmemdup(wal->list,
  68                                                wal->count * sizeof(*list),
  69                                                GFP_KERNEL);
  70
  71                 if (list) {
  72                         kfree(wal->list);
  73                         wal->list = list;
  74                 }
  75         }
  76
  77         if (!wal->count)
  78                 return;
  79
  80         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
  81                          wal->wa_count, wal->name, wal->engine_name);
  82 }
  83
  84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  85 {
  86         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  87         unsigned int start = 0, end = wal->count;
  88         const unsigned int grow = WA_LIST_CHUNK;
  89         struct i915_wa *wa_;
  90
  91         GEM_BUG_ON(!is_power_of_2(grow));
  92
  93         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  94                 struct i915_wa *list;
  95
  96                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
  97                                      GFP_KERNEL);
  98                 if (!list) {
  99                         DRM_ERROR("No space for workaround init!\n");
 100                         return;
 101                 }
 102
 103                 if (wal->list)
 104                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 105
 106                 wal->list = list;
 107         }
 108
 109         while (start < end) {
 110                 unsigned int mid = start + (end - start) / 2;
 111
 112                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 113                         start = mid + 1;
 114                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 115                         end = mid;
 116                 } else {
 117                         wa_ = &wal->list[mid];
 118
 119                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 120                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 121                                           i915_mmio_reg_offset(wa_->reg),
 122                                           wa_->clr, wa_->set);
 123
 124                                 wa_->set &= ~wa->clr;
 125                         }
 126
 127                         wal->wa_count++;
 128                         wa_->set |= wa->set;
 129                         wa_->clr |= wa->clr;
 130                         wa_->read |= wa->read;
 131                         return;
 132                 }
 133         }
 134
 135         wal->wa_count++;
 136         wa_ = &wal->list[wal->count++];
 137         *wa_ = *wa;
 138
 139         while (wa_-- > wal->list) {
 140                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 141                            i915_mmio_reg_offset(wa_[1].reg));
 142                 if (i915_mmio_reg_offset(wa_[1].reg) >
 143                     i915_mmio_reg_offset(wa_[0].reg))
 144                         break;
 145
 146                 swap(wa_[1], wa_[0]);
 147         }
 148 }
 149
 150 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 151                    u32 clear, u32 set, u32 read_mask)
 152 {
 153         struct i915_wa wa = {
 154                 .reg  = reg,
 155                 .clr  = clear,
 156                 .set  = set,
 157                 .read = read_mask,
 158         };
 159
 160         _wa_add(wal, &wa);
 161 }
 162
 163 static void
 164 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 165 {
 166         wa_add(wal, reg, clear, set, clear);
 167 }
 168
 169 static void
 170 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 171 {
 172         wa_write_masked_or(wal, reg, ~0, set);
 173 }
 174
 175 static void
 176 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 177 {
 178         wa_write_masked_or(wal, reg, set, set);
 179 }
 180
 181 static void
 182 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
 183 {
 184         wa_write_masked_or(wal, reg, clr, 0);
 185 }
 186
 187 static void
 188 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 189 {
 190         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
 191 }
 192
 193 static void
 194 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 195 {
 196         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
 197 }
 198
 199 #define WA_SET_BIT_MASKED(addr, mask) \
 200         wa_masked_en(wal, (addr), (mask))
 201
 202 #define WA_CLR_BIT_MASKED(addr, mask) \
 203         wa_masked_dis(wal, (addr), (mask))
 204
 205 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 206         wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
 207
 208 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 209                                       struct i915_wa_list *wal)
 210 {
 211         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 212
 213         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 214         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 215
 216         /* WaDisablePartialInstShootdown:bdw,chv */
 217         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 218                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 219
 220         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 221          * workaround for for a possible hang in the unlikely event a TLB
 222          * invalidation occurs during a PSD flush.
 223          */
 224         /* WaForceEnableNonCoherent:bdw,chv */
 225         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 226         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 227                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 228                           HDC_FORCE_NON_COHERENT);
 229
 230         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 231          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 232          *  polygons in the same 8x4 pixel/sample area to be processed without
 233          *  stalling waiting for the earlier ones to write to Hierarchical Z
 234          *  buffer."
 235          *
 236          * This optimization is off by default for BDW and CHV; turn it on.
 237          */
 238         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 239
 240         /* Wa4x4STCOptimizationDisable:bdw,chv */
 241         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 242
 243         /*
 244          * BSpec recommends 8x4 when MSAA is used,
 245          * however in practice 16x4 seems fastest.
 246          *
 247          * Note that PS/WM thread counts depend on the WIZ hashing
 248          * disable bit, which we don't touch here, but it's good
 249          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 250          */
 251         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 252                             GEN6_WIZ_HASHING_MASK,
 253                             GEN6_WIZ_HASHING_16x4);
 254 }
 255
 256 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 257                                      struct i915_wa_list *wal)
 258 {
 259         struct drm_i915_private *i915 = engine->i915;
 260
 261         gen8_ctx_workarounds_init(engine, wal);
 262
 263         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 264         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 265
 266         /* WaDisableDopClockGating:bdw
 267          *
 268          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 269          * to disable EUTC clock gating.
 270          */
 271         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 272                           DOP_CLOCK_GATING_DISABLE);
 273
 274         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 275                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 276
 277         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 278                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 279                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 280                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 281                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 282 }
 283
 284 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 285                                      struct i915_wa_list *wal)
 286 {
 287         gen8_ctx_workarounds_init(engine, wal);
 288
 289         /* WaDisableThreadStallDopClockGating:chv */
 290         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 291
 292         /* Improve HiZ throughput on CHV. */
 293         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 294 }
 295
 296 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 297                                       struct i915_wa_list *wal)
 298 {
 299         struct drm_i915_private *i915 = engine->i915;
 300
 301         if (HAS_LLC(i915)) {
 302                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 303                  *
 304                  * Must match Display Engine. See
 305                  * WaCompressedResourceDisplayNewHashMode.
 306                  */
 307                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 308                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 309                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 310                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 311         }
 312
 313         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 314         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 315         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 316                           FLOW_CONTROL_ENABLE |
 317                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 318
 319         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 320         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 321         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 322                           GEN9_ENABLE_YV12_BUGFIX |
 323                           GEN9_ENABLE_GPGPU_PREEMPTION);
 324
 325         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 326         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 327         WA_SET_BIT_MASKED(CACHE_MODE_1,
 328                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 329                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 330
 331         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 332         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 333                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 334
 335         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 336         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 337                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 338                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 339
 340         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 341          * both tied to WaForceContextSaveRestoreNonCoherent
 342          * in some hsds for skl. We keep the tie for all gen9. The
 343          * documentation is a bit hazy and so we want to get common behaviour,
 344          * even though there is no clear evidence we would need both on kbl/bxt.
 345          * This area has been source of system hangs so we play it safe
 346          * and mimic the skl regardless of what bspec says.
 347          *
 348          * Use Force Non-Coherent whenever executing a 3D context. This
 349          * is a workaround for a possible hang in the unlikely event
 350          * a TLB invalidation occurs during a PSD flush.
 351          */
 352
 353         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 354         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 355                           HDC_FORCE_NON_COHERENT);
 356
 357         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 358         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
 359                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 360                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 361
 362         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 363         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 364
 365         /*
 366          * Supporting preemption with fine-granularity requires changes in the
 367          * batch buffer programming. Since we can't break old userspace, we
 368          * need to set our default preemption level to safe value. Userspace is
 369          * still able to use more fine-grained preemption levels, since in
 370          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 371          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 372          * not real HW workarounds, but merely a way to start using preemption
 373          * while maintaining old contract with userspace.
 374          */
 375
 376         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 377         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 378
 379         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 380         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 381                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 382                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 383
 384         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 385         if (IS_GEN9_LP(i915))
 386                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 387 }
 388
 389 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 390                                 struct i915_wa_list *wal)
 391 {
 392         struct drm_i915_private *i915 = engine->i915;
 393         u8 vals[3] = { 0, 0, 0 };
 394         unsigned int i;
 395
 396         for (i = 0; i < 3; i++) {
 397                 u8 ss;
 398
 399                 /*
 400                  * Only consider slices where one, and only one, subslice has 7
 401                  * EUs
 402                  */
 403                 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
 404                         continue;
 405
 406                 /*
 407                  * subslice_7eu[i] != 0 (because of the check above) and
 408                  * ss_max == 4 (maximum number of subslices possible per slice)
 409                  *
 410                  * ->    0 <= ss <= 3;
 411                  */
 412                 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
 413                 vals[i] = 3 - ss;
 414         }
 415
 416         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 417                 return;
 418
 419         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 420         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 421                             GEN9_IZ_HASHING_MASK(2) |
 422                             GEN9_IZ_HASHING_MASK(1) |
 423                             GEN9_IZ_HASHING_MASK(0),
 424                             GEN9_IZ_HASHING(2, vals[2]) |
 425                             GEN9_IZ_HASHING(1, vals[1]) |
 426                             GEN9_IZ_HASHING(0, vals[0]));
 427 }
 428
 429 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 430                                      struct i915_wa_list *wal)
 431 {
 432         gen9_ctx_workarounds_init(engine, wal);
 433         skl_tune_iz_hashing(engine, wal);
 434 }
 435
 436 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 437                                      struct i915_wa_list *wal)
 438 {
 439         gen9_ctx_workarounds_init(engine, wal);
 440
 441         /* WaDisableThreadStallDopClockGating:bxt */
 442         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 443                           STALL_DOP_GATING_DISABLE);
 444
 445         /* WaToEnableHwFixForPushConstHWBug:bxt */
 446         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 447                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 448 }
 449
 450 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 451                                      struct i915_wa_list *wal)
 452 {
 453         struct drm_i915_private *i915 = engine->i915;
 454
 455         gen9_ctx_workarounds_init(engine, wal);
 456
 457         /* WaToEnableHwFixForPushConstHWBug:kbl */
 458         if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 459                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 460                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 461
 462         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 463         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 464                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 465 }
 466
 467 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 468                                      struct i915_wa_list *wal)
 469 {
 470         gen9_ctx_workarounds_init(engine, wal);
 471
 472         /* WaToEnableHwFixForPushConstHWBug:glk */
 473         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 474                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 475 }
 476
 477 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 478                                      struct i915_wa_list *wal)
 479 {
 480         gen9_ctx_workarounds_init(engine, wal);
 481
 482         /* WaToEnableHwFixForPushConstHWBug:cfl */
 483         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 484                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 485
 486         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 487         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 488                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 489 }
 490
 491 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 492                                      struct i915_wa_list *wal)
 493 {
 494         /* WaForceContextSaveRestoreNonCoherent:cnl */
 495         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 496                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 497
 498         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 499         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 500                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 501
 502         /* WaPushConstantDereferenceHoldDisable:cnl */
 503         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 504
 505         /* FtrEnableFastAnisoL1BankingFix:cnl */
 506         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 507
 508         /* WaDisable3DMidCmdPreemption:cnl */
 509         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 510
 511         /* WaDisableGPGPUMidCmdPreemption:cnl */
 512         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 513                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 514                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 515
 516         /* WaDisableEarlyEOT:cnl */
 517         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 518 }
 519
 520 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 521                                      struct i915_wa_list *wal)
 522 {
 523         struct drm_i915_private *i915 = engine->i915;
 524
 525         /* WaDisableBankHangMode:icl */
 526         wa_write(wal,
 527                  GEN8_L3CNTLREG,
 528                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 529                  GEN8_ERRDETBCTRL);
 530
 531         /* Wa_1604370585:icl (pre-prod)
 532          * Formerly known as WaPushConstantDereferenceHoldDisable
 533          */
 534         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 535                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 536                                   PUSH_CONSTANT_DEREF_DISABLE);
 537
 538         /* WaForceEnableNonCoherent:icl
 539          * This is not the same workaround as in early Gen9 platforms, where
 540          * lacking this could cause system hangs, but coherency performance
 541          * overhead is high and only a few compute workloads really need it
 542          * (the register is whitelisted in hardware now, so UMDs can opt in
 543          * for coherency if they have a good reason).
 544          */
 545         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 546
 547         /* Wa_2006611047:icl (pre-prod)
 548          * Formerly known as WaDisableImprovedTdlClkGating
 549          */
 550         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 551                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 552                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 553
 554         /* Wa_2006665173:icl (pre-prod) */
 555         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 556                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 557                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 558
 559         /* WaEnableFloatBlendOptimization:icl */
 560         wa_write_masked_or(wal,
 561                            GEN10_CACHE_MODE_SS,
 562                            0, /* write-only, so skip validation */
 563                            _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 564
 565         /* WaDisableGPGPUMidThreadPreemption:icl */
 566         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 567                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 568                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 569
 570         /* allow headerless messages for preemptible GPGPU context */
 571         WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
 572                           GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 573
 574         /* Wa_1604278689:icl,ehl */
 575         wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
 576         wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER,
 577                            0, /* write-only register; skip validation */
 578                            0xFFFFFFFF);
 579
 580         /* Wa_1406306137:icl,ehl */
 581         wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
 582 }
 583
 584 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
 585                                      struct i915_wa_list *wal)
 586 {
 587         /*
 588          * Wa_1409142259:tgl
 589          * Wa_1409347922:tgl
 590          * Wa_1409252684:tgl
 591          * Wa_1409217633:tgl
 592          * Wa_1409207793:tgl
 593          * Wa_1409178076:tgl
 594          * Wa_1408979724:tgl
 595          */
 596         WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 597                           GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 598
 599         /*
 600          * Wa_1604555607:gen12 and Wa_1608008084:gen12
 601          * FF_MODE2 register will return the wrong value when read. The default
 602          * value for this register is zero for all fields and there are no bit
 603          * masks. So instead of doing a RMW we should just write the TDS timer
 604          * value for Wa_1604555607.
 605          */
 606         wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK,
 607                FF_MODE2_TDS_TIMER_128, 0);
 608
 609         /* WaDisableGPGPUMidThreadPreemption:tgl */
 610         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 611                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 612                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 613 }
 614
 615 static void
 616 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 617                            struct i915_wa_list *wal,
 618                            const char *name)
 619 {
 620         struct drm_i915_private *i915 = engine->i915;
 621
 622         if (engine->class != RENDER_CLASS)
 623                 return;
 624
 625         wa_init_start(wal, name, engine->name);
 626
 627         if (IS_GEN(i915, 12))
 628                 tgl_ctx_workarounds_init(engine, wal);
 629         else if (IS_GEN(i915, 11))
 630                 icl_ctx_workarounds_init(engine, wal);
 631         else if (IS_CANNONLAKE(i915))
 632                 cnl_ctx_workarounds_init(engine, wal);
 633         else if (IS_COFFEELAKE(i915))
 634                 cfl_ctx_workarounds_init(engine, wal);
 635         else if (IS_GEMINILAKE(i915))
 636                 glk_ctx_workarounds_init(engine, wal);
 637         else if (IS_KABYLAKE(i915))
 638                 kbl_ctx_workarounds_init(engine, wal);
 639         else if (IS_BROXTON(i915))
 640                 bxt_ctx_workarounds_init(engine, wal);
 641         else if (IS_SKYLAKE(i915))
 642                 skl_ctx_workarounds_init(engine, wal);
 643         else if (IS_CHERRYVIEW(i915))
 644                 chv_ctx_workarounds_init(engine, wal);
 645         else if (IS_BROADWELL(i915))
 646                 bdw_ctx_workarounds_init(engine, wal);
 647         else if (INTEL_GEN(i915) < 8)
 648                 return;
 649         else
 650                 MISSING_CASE(INTEL_GEN(i915));
 651
 652         wa_init_finish(wal);
 653 }
 654
 655 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 656 {
 657         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 658 }
 659
 660 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 661 {
 662         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 663         struct i915_wa *wa;
 664         unsigned int i;
 665         u32 *cs;
 666         int ret;
 667
 668         if (wal->count == 0)
 669                 return 0;
 670
 671         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 672         if (ret)
 673                 return ret;
 674
 675         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 676         if (IS_ERR(cs))
 677                 return PTR_ERR(cs);
 678
 679         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 680         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 681                 *cs++ = i915_mmio_reg_offset(wa->reg);
 682                 *cs++ = wa->set;
 683         }
 684         *cs++ = MI_NOOP;
 685
 686         intel_ring_advance(rq, cs);
 687
 688         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 689         if (ret)
 690                 return ret;
 691
 692         return 0;
 693 }
 694
 695 static void
 696 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 697 {
 698         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
 699         wa_masked_en(wal,
 700                      _3D_CHICKEN,
 701                      _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
 702
 703         /* WaDisable_RenderCache_OperationalFlush:snb */
 704         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 705
 706         /*
 707          * BSpec recommends 8x4 when MSAA is used,
 708          * however in practice 16x4 seems fastest.
 709          *
 710          * Note that PS/WM thread counts depend on the WIZ hashing
 711          * disable bit, which we don't touch here, but it's good
 712          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 713          */
 714         wa_add(wal,
 715                GEN6_GT_MODE, 0,
 716                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 717                GEN6_WIZ_HASHING_16x4);
 718
 719         wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB);
 720
 721         wa_masked_en(wal,
 722                      _3D_CHICKEN3,
 723                      /* WaStripsFansDisableFastClipPerformanceFix:snb */
 724                      _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
 725                      /*
 726                       * Bspec says:
 727                       * "This bit must be set if 3DSTATE_CLIP clip mode is set
 728                       * to normal and 3DSTATE_SF number of SF output attributes
 729                       * is more than 16."
 730                       */
 731                    _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
 732 }
 733
 734 static void
 735 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 736 {
 737         /* WaDisableEarlyCull:ivb */
 738         wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
 739
 740         /* WaDisablePSDDualDispatchEnable:ivb */
 741         if (IS_IVB_GT1(i915))
 742                 wa_masked_en(wal,
 743                              GEN7_HALF_SLICE_CHICKEN1,
 744                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
 745
 746         /* WaDisable_RenderCache_OperationalFlush:ivb */
 747         wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
 748
 749         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 750         wa_masked_dis(wal,
 751                       GEN7_COMMON_SLICE_CHICKEN1,
 752                       GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 753
 754         /* WaApplyL3ControlAndL3ChickenMode:ivb */
 755         wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
 756         wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 757
 758         /* WaForceL3Serialization:ivb */
 759         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 760
 761         /*
 762          * WaVSThreadDispatchOverride:ivb,vlv
 763          *
 764          * This actually overrides the dispatch
 765          * mode for all thread types.
 766          */
 767         wa_write_masked_or(wal, GEN7_FF_THREAD_MODE,
 768                            GEN7_FF_SCHED_MASK,
 769                            GEN7_FF_TS_SCHED_HW |
 770                            GEN7_FF_VS_SCHED_HW |
 771                            GEN7_FF_DS_SCHED_HW);
 772
 773         if (0) { /* causes HiZ corruption on ivb:gt1 */
 774                 /* enable HiZ Raw Stall Optimization */
 775                 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 776         }
 777
 778         /* WaDisable4x2SubspanOptimization:ivb */
 779         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 780
 781         /*
 782          * BSpec recommends 8x4 when MSAA is used,
 783          * however in practice 16x4 seems fastest.
 784          *
 785          * Note that PS/WM thread counts depend on the WIZ hashing
 786          * disable bit, which we don't touch here, but it's good
 787          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 788          */
 789         wa_add(wal, GEN7_GT_MODE, 0,
 790                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 791                GEN6_WIZ_HASHING_16x4);
 792 }
 793
 794 static void
 795 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 796 {
 797         /* WaDisableEarlyCull:vlv */
 798         wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
 799
 800         /* WaPsdDispatchEnable:vlv */
 801         /* WaDisablePSDDualDispatchEnable:vlv */
 802         wa_masked_en(wal,
 803                      GEN7_HALF_SLICE_CHICKEN1,
 804                      GEN7_MAX_PS_THREAD_DEP |
 805                      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
 806
 807         /* WaDisable_RenderCache_OperationalFlush:vlv */
 808         wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
 809
 810         /* WaForceL3Serialization:vlv */
 811         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 812
 813         /*
 814          * WaVSThreadDispatchOverride:ivb,vlv
 815          *
 816          * This actually overrides the dispatch
 817          * mode for all thread types.
 818          */
 819         wa_write_masked_or(wal,
 820                            GEN7_FF_THREAD_MODE,
 821                            GEN7_FF_SCHED_MASK,
 822                            GEN7_FF_TS_SCHED_HW |
 823                            GEN7_FF_VS_SCHED_HW |
 824                            GEN7_FF_DS_SCHED_HW);
 825
 826         /*
 827          * BSpec says this must be set, even though
 828          * WaDisable4x2SubspanOptimization isn't listed for VLV.
 829          */
 830         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 831
 832         /*
 833          * BSpec recommends 8x4 when MSAA is used,
 834          * however in practice 16x4 seems fastest.
 835          *
 836          * Note that PS/WM thread counts depend on the WIZ hashing
 837          * disable bit, which we don't touch here, but it's good
 838          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 839          */
 840         wa_add(wal, GEN7_GT_MODE, 0,
 841                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 842                GEN6_WIZ_HASHING_16x4);
 843
 844         /*
 845          * WaIncreaseL3CreditsForVLVB0:vlv
 846          * This is the hardware default actually.
 847          */
 848         wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
 849 }
 850
 851 static void
 852 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 853 {
 854         /* L3 caching of data atomics doesn't work -- disable it. */
 855         wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
 856
 857         wa_add(wal,
 858                HSW_ROW_CHICKEN3, 0,
 859                _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
 860                 0 /* XXX does this reg exist? */);
 861
 862         /* WaVSRefCountFullforceMissDisable:hsw */
 863         wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
 864
 865         wa_masked_dis(wal,
 866                       CACHE_MODE_0_GEN7,
 867                       /* WaDisable_RenderCache_OperationalFlush:hsw */
 868                       RC_OP_FLUSH_ENABLE |
 869                       /* enable HiZ Raw Stall Optimization */
 870                       HIZ_RAW_STALL_OPT_DISABLE);
 871
 872         /* WaDisable4x2SubspanOptimization:hsw */
 873         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 874
 875         /*
 876          * BSpec recommends 8x4 when MSAA is used,
 877          * however in practice 16x4 seems fastest.
 878          *
 879          * Note that PS/WM thread counts depend on the WIZ hashing
 880          * disable bit, which we don't touch here, but it's good
 881          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 882          */
 883         wa_add(wal, GEN7_GT_MODE, 0,
 884                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 885                GEN6_WIZ_HASHING_16x4);
 886
 887         /* WaSampleCChickenBitEnable:hsw */
 888         wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
 889 }
 890
 891 static void
 892 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 893 {
 894         /* WaDisableKillLogic:bxt,skl,kbl */
 895         if (!IS_COFFEELAKE(i915))
 896                 wa_write_or(wal,
 897                             GAM_ECOCHK,
 898                             ECOCHK_DIS_TLB);
 899
 900         if (HAS_LLC(i915)) {
 901                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 902                  *
 903                  * Must match Display Engine. See
 904                  * WaCompressedResourceDisplayNewHashMode.
 905                  */
 906                 wa_write_or(wal,
 907                             MMCD_MISC_CTRL,
 908                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 909         }
 910
 911         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
 912         wa_write_or(wal,
 913                     GAM_ECOCHK,
 914                     BDW_DISABLE_HDC_INVALIDATION);
 915 }
 916
 917 static void
 918 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 919 {
 920         gen9_gt_workarounds_init(i915, wal);
 921
 922         /* WaDisableGafsUnitClkGating:skl */
 923         wa_write_or(wal,
 924                     GEN7_UCGCTL4,
 925                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 926
 927         /* WaInPlaceDecompressionHang:skl */
 928         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
 929                 wa_write_or(wal,
 930                             GEN9_GAMT_ECO_REG_RW_IA,
 931                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 932 }
 933
 934 static void
 935 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 936 {
 937         gen9_gt_workarounds_init(i915, wal);
 938
 939         /* WaInPlaceDecompressionHang:bxt */
 940         wa_write_or(wal,
 941                     GEN9_GAMT_ECO_REG_RW_IA,
 942                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 943 }
 944
 945 static void
 946 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 947 {
 948         gen9_gt_workarounds_init(i915, wal);
 949
 950         /* WaDisableDynamicCreditSharing:kbl */
 951         if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
 952                 wa_write_or(wal,
 953                             GAMT_CHKN_BIT_REG,
 954                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
 955
 956         /* WaDisableGafsUnitClkGating:kbl */
 957         wa_write_or(wal,
 958                     GEN7_UCGCTL4,
 959                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 960
 961         /* WaInPlaceDecompressionHang:kbl */
 962         wa_write_or(wal,
 963                     GEN9_GAMT_ECO_REG_RW_IA,
 964                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 965 }
 966
 967 static void
 968 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 969 {
 970         gen9_gt_workarounds_init(i915, wal);
 971 }
 972
 973 static void
 974 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 975 {
 976         gen9_gt_workarounds_init(i915, wal);
 977
 978         /* WaDisableGafsUnitClkGating:cfl */
 979         wa_write_or(wal,
 980                     GEN7_UCGCTL4,
 981                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 982
 983         /* WaInPlaceDecompressionHang:cfl */
 984         wa_write_or(wal,
 985                     GEN9_GAMT_ECO_REG_RW_IA,
 986                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 987 }
 988
 989 static void
 990 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 991 {
 992         const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
 993         unsigned int slice, subslice;
 994         u32 l3_en, mcr, mcr_mask;
 995
 996         GEM_BUG_ON(INTEL_GEN(i915) < 10);
 997
 998         /*
 999          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
1000          * L3Banks could be fused off in single slice scenario. If that is
1001          * the case, we might need to program MCR select to a valid L3Bank
1002          * by default, to make sure we correctly read certain registers
1003          * later on (in the range 0xB100 - 0xB3FF).
1004          *
1005          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
1006          * Before any MMIO read into slice/subslice specific registers, MCR
1007          * packet control register needs to be programmed to point to any
1008          * enabled s/ss pair. Otherwise, incorrect values will be returned.
1009          * This means each subsequent MMIO read will be forwarded to an
1010          * specific s/ss combination, but this is OK since these registers
1011          * are consistent across s/ss in almost all cases. In the rare
1012          * occasions, such as INSTDONE, where this value is dependent
1013          * on s/ss combo, the read should be done with read_subslice_reg.
1014          *
1015          * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
1016          * to which subslice, or to which L3 bank, the respective mmio reads
1017          * will go, we have to find a common index which works for both
1018          * accesses.
1019          *
1020          * Case where we cannot find a common index fortunately should not
1021          * happen in production hardware, so we only emit a warning instead of
1022          * implementing something more complex that requires checking the range
1023          * of every MMIO read.
1024          */
1025
1026         if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
1027                 u32 l3_fuse =
1028                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
1029                         GEN10_L3BANK_MASK;
1030
1031                 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
1032                 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
1033         } else {
1034                 l3_en = ~0;
1035         }
1036
1037         slice = fls(sseu->slice_mask) - 1;
1038         subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
1039         if (!subslice) {
1040                 drm_warn(&i915->drm,
1041                          "No common index found between subslice mask %x and L3 bank mask %x!\n",
1042                          intel_sseu_get_subslices(sseu, slice), l3_en);
1043                 subslice = fls(l3_en);
1044                 drm_WARN_ON(&i915->drm, !subslice);
1045         }
1046         subslice--;
1047
1048         if (INTEL_GEN(i915) >= 11) {
1049                 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1050                 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1051         } else {
1052                 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1053                 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1054         }
1055
1056         drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1057
1058         wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1059 }
1060
1061 static void
1062 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1063 {
1064         wa_init_mcr(i915, wal);
1065
1066         /* WaInPlaceDecompressionHang:cnl */
1067         wa_write_or(wal,
1068                     GEN9_GAMT_ECO_REG_RW_IA,
1069                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1070 }
1071
1072 static void
1073 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1074 {
1075         wa_init_mcr(i915, wal);
1076
1077         /* WaInPlaceDecompressionHang:icl */
1078         wa_write_or(wal,
1079                     GEN9_GAMT_ECO_REG_RW_IA,
1080                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1081
1082         /* WaModifyGamTlbPartitioning:icl */
1083         wa_write_masked_or(wal,
1084                            GEN11_GACB_PERF_CTRL,
1085                            GEN11_HASH_CTRL_MASK,
1086                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1087
1088         /* Wa_1405766107:icl
1089          * Formerly known as WaCL2SFHalfMaxAlloc
1090          */
1091         wa_write_or(wal,
1092                     GEN11_LSN_UNSLCVC,
1093                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1094                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1095
1096         /* Wa_220166154:icl
1097          * Formerly known as WaDisCtxReload
1098          */
1099         wa_write_or(wal,
1100                     GEN8_GAMW_ECO_DEV_RW_IA,
1101                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1102
1103         /* Wa_1405779004:icl (pre-prod) */
1104         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1105                 wa_write_or(wal,
1106                             SLICE_UNIT_LEVEL_CLKGATE,
1107                             MSCUNIT_CLKGATE_DIS);
1108
1109         /* Wa_1406838659:icl (pre-prod) */
1110         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1111                 wa_write_or(wal,
1112                             INF_UNIT_LEVEL_CLKGATE,
1113                             CGPSF_CLKGATE_DIS);
1114
1115         /* Wa_1406463099:icl
1116          * Formerly known as WaGamTlbPendError
1117          */
1118         wa_write_or(wal,
1119                     GAMT_CHKN_BIT_REG,
1120                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
1121
1122         /* Wa_1607087056:icl,ehl,jsl */
1123         if (IS_ICELAKE(i915) ||
1124             IS_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) {
1125                 wa_write_or(wal,
1126                             SLICE_UNIT_LEVEL_CLKGATE,
1127                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1128         }
1129 }
1130
1131 static void
1132 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1133 {
1134         wa_init_mcr(i915, wal);
1135
1136         /* Wa_1409420604:tgl */
1137         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1138                 wa_write_or(wal,
1139                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1140                             CPSSUNIT_CLKGATE_DIS);
1141
1142         /* Wa_1607087056:tgl also know as BUG:1409180338 */
1143         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1144                 wa_write_or(wal,
1145                             SLICE_UNIT_LEVEL_CLKGATE,
1146                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1147 }
1148
1149 static void
1150 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1151 {
1152         if (IS_GEN(i915, 12))
1153                 tgl_gt_workarounds_init(i915, wal);
1154         else if (IS_GEN(i915, 11))
1155                 icl_gt_workarounds_init(i915, wal);
1156         else if (IS_CANNONLAKE(i915))
1157                 cnl_gt_workarounds_init(i915, wal);
1158         else if (IS_COFFEELAKE(i915))
1159                 cfl_gt_workarounds_init(i915, wal);
1160         else if (IS_GEMINILAKE(i915))
1161                 glk_gt_workarounds_init(i915, wal);
1162         else if (IS_KABYLAKE(i915))
1163                 kbl_gt_workarounds_init(i915, wal);
1164         else if (IS_BROXTON(i915))
1165                 bxt_gt_workarounds_init(i915, wal);
1166         else if (IS_SKYLAKE(i915))
1167                 skl_gt_workarounds_init(i915, wal);
1168         else if (IS_HASWELL(i915))
1169                 hsw_gt_workarounds_init(i915, wal);
1170         else if (IS_VALLEYVIEW(i915))
1171                 vlv_gt_workarounds_init(i915, wal);
1172         else if (IS_IVYBRIDGE(i915))
1173                 ivb_gt_workarounds_init(i915, wal);
1174         else if (IS_GEN(i915, 6))
1175                 snb_gt_workarounds_init(i915, wal);
1176         else if (INTEL_GEN(i915) <= 8)
1177                 return;
1178         else
1179                 MISSING_CASE(INTEL_GEN(i915));
1180 }
1181
1182 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1183 {
1184         struct i915_wa_list *wal = &i915->gt_wa_list;
1185
1186         wa_init_start(wal, "GT", "global");
1187         gt_init_workarounds(i915, wal);
1188         wa_init_finish(wal);
1189 }
1190
1191 static enum forcewake_domains
1192 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1193 {
1194         enum forcewake_domains fw = 0;
1195         struct i915_wa *wa;
1196         unsigned int i;
1197
1198         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1199                 fw |= intel_uncore_forcewake_for_reg(uncore,
1200                                                      wa->reg,
1201                                                      FW_REG_READ |
1202                                                      FW_REG_WRITE);
1203
1204         return fw;
1205 }
1206
1207 static bool
1208 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1209 {
1210         if ((cur ^ wa->set) & wa->read) {
1211                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
1212                           name, from, i915_mmio_reg_offset(wa->reg),
1213                           cur, cur & wa->read, wa->set);
1214
1215                 return false;
1216         }
1217
1218         return true;
1219 }
1220
1221 static void
1222 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1223 {
1224         enum forcewake_domains fw;
1225         unsigned long flags;
1226         struct i915_wa *wa;
1227         unsigned int i;
1228
1229         if (!wal->count)
1230                 return;
1231
1232         fw = wal_get_fw_for_rmw(uncore, wal);
1233
1234         spin_lock_irqsave(&uncore->lock, flags);
1235         intel_uncore_forcewake_get__locked(uncore, fw);
1236
1237         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1238                 if (wa->clr)
1239                         intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1240                 else
1241                         intel_uncore_write_fw(uncore, wa->reg, wa->set);
1242                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1243                         wa_verify(wa,
1244                                   intel_uncore_read_fw(uncore, wa->reg),
1245                                   wal->name, "application");
1246         }
1247
1248         intel_uncore_forcewake_put__locked(uncore, fw);
1249         spin_unlock_irqrestore(&uncore->lock, flags);
1250 }
1251
1252 void intel_gt_apply_workarounds(struct intel_gt *gt)
1253 {
1254         wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1255 }
1256
1257 static bool wa_list_verify(struct intel_uncore *uncore,
1258                            const struct i915_wa_list *wal,
1259                            const char *from)
1260 {
1261         struct i915_wa *wa;
1262         unsigned int i;
1263         bool ok = true;
1264
1265         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1266                 ok &= wa_verify(wa,
1267                                 intel_uncore_read(uncore, wa->reg),
1268                                 wal->name, from);
1269
1270         return ok;
1271 }
1272
1273 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1274 {
1275         return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1276 }
1277
1278 static inline bool is_nonpriv_flags_valid(u32 flags)
1279 {
1280         /* Check only valid flag bits are set */
1281         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1282                 return false;
1283
1284         /* NB: Only 3 out of 4 enum values are valid for access field */
1285         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1286             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1287                 return false;
1288
1289         return true;
1290 }
1291
1292 static void
1293 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1294 {
1295         struct i915_wa wa = {
1296                 .reg = reg
1297         };
1298
1299         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1300                 return;
1301
1302         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1303                 return;
1304
1305         wa.reg.reg |= flags;
1306         _wa_add(wal, &wa);
1307 }
1308
1309 static void
1310 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1311 {
1312         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1313 }
1314
1315 static void gen9_whitelist_build(struct i915_wa_list *w)
1316 {
1317         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1318         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1319
1320         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1321         whitelist_reg(w, GEN8_CS_CHICKEN1);
1322
1323         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1324         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1325
1326         /* WaSendPushConstantsFromMMIO:skl,bxt */
1327         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1328 }
1329
1330 static void skl_whitelist_build(struct intel_engine_cs *engine)
1331 {
1332         struct i915_wa_list *w = &engine->whitelist;
1333
1334         if (engine->class != RENDER_CLASS)
1335                 return;
1336
1337         gen9_whitelist_build(w);
1338
1339         /* WaDisableLSQCROPERFforOCL:skl */
1340         whitelist_reg(w, GEN8_L3SQCREG4);
1341 }
1342
1343 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1344 {
1345         if (engine->class != RENDER_CLASS)
1346                 return;
1347
1348         gen9_whitelist_build(&engine->whitelist);
1349 }
1350
1351 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1352 {
1353         struct i915_wa_list *w = &engine->whitelist;
1354
1355         if (engine->class != RENDER_CLASS)
1356                 return;
1357
1358         gen9_whitelist_build(w);
1359
1360         /* WaDisableLSQCROPERFforOCL:kbl */
1361         whitelist_reg(w, GEN8_L3SQCREG4);
1362 }
1363
1364 static void glk_whitelist_build(struct intel_engine_cs *engine)
1365 {
1366         struct i915_wa_list *w = &engine->whitelist;
1367
1368         if (engine->class != RENDER_CLASS)
1369                 return;
1370
1371         gen9_whitelist_build(w);
1372
1373         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1374         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1375 }
1376
1377 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1378 {
1379         struct i915_wa_list *w = &engine->whitelist;
1380
1381         if (engine->class != RENDER_CLASS)
1382                 return;
1383
1384         gen9_whitelist_build(w);
1385
1386         /*
1387          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1388          *
1389          * This covers 4 register which are next to one another :
1390          *   - PS_INVOCATION_COUNT
1391          *   - PS_INVOCATION_COUNT_UDW
1392          *   - PS_DEPTH_COUNT
1393          *   - PS_DEPTH_COUNT_UDW
1394          */
1395         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1396                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1397                           RING_FORCE_TO_NONPRIV_RANGE_4);
1398 }
1399
1400 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1401 {
1402         struct i915_wa_list *w = &engine->whitelist;
1403
1404         if (engine->class != RENDER_CLASS)
1405                 return;
1406
1407         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1408         whitelist_reg(w, GEN8_CS_CHICKEN1);
1409 }
1410
1411 static void icl_whitelist_build(struct intel_engine_cs *engine)
1412 {
1413         struct i915_wa_list *w = &engine->whitelist;
1414
1415         switch (engine->class) {
1416         case RENDER_CLASS:
1417                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1418                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1419
1420                 /* WaAllowUMDToModifySamplerMode:icl */
1421                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1422
1423                 /* WaEnableStateCacheRedirectToCS:icl */
1424                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1425
1426                 /*
1427                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1428                  *
1429                  * This covers 4 register which are next to one another :
1430                  *   - PS_INVOCATION_COUNT
1431                  *   - PS_INVOCATION_COUNT_UDW
1432                  *   - PS_DEPTH_COUNT
1433                  *   - PS_DEPTH_COUNT_UDW
1434                  */
1435                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1436                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1437                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1438                 break;
1439
1440         case VIDEO_DECODE_CLASS:
1441                 /* hucStatusRegOffset */
1442                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1443                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1444                 /* hucUKernelHdrInfoRegOffset */
1445                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1446                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1447                 /* hucStatus2RegOffset */
1448                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1449                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1450                 break;
1451
1452         default:
1453                 break;
1454         }
1455 }
1456
1457 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1458 {
1459         struct i915_wa_list *w = &engine->whitelist;
1460
1461         switch (engine->class) {
1462         case RENDER_CLASS:
1463                 /*
1464                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1465                  * Wa_1408556865:tgl
1466                  *
1467                  * This covers 4 registers which are next to one another :
1468                  *   - PS_INVOCATION_COUNT
1469                  *   - PS_INVOCATION_COUNT_UDW
1470                  *   - PS_DEPTH_COUNT
1471                  *   - PS_DEPTH_COUNT_UDW
1472                  */
1473                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1474                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1475                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1476
1477                 /* Wa_1808121037:tgl */
1478                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1479
1480                 /* Wa_1806527549:tgl */
1481                 whitelist_reg(w, HIZ_CHICKEN);
1482                 break;
1483         default:
1484                 break;
1485         }
1486 }
1487
1488 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1489 {
1490         struct drm_i915_private *i915 = engine->i915;
1491         struct i915_wa_list *w = &engine->whitelist;
1492
1493         wa_init_start(w, "whitelist", engine->name);
1494
1495         if (IS_GEN(i915, 12))
1496                 tgl_whitelist_build(engine);
1497         else if (IS_GEN(i915, 11))
1498                 icl_whitelist_build(engine);
1499         else if (IS_CANNONLAKE(i915))
1500                 cnl_whitelist_build(engine);
1501         else if (IS_COFFEELAKE(i915))
1502                 cfl_whitelist_build(engine);
1503         else if (IS_GEMINILAKE(i915))
1504                 glk_whitelist_build(engine);
1505         else if (IS_KABYLAKE(i915))
1506                 kbl_whitelist_build(engine);
1507         else if (IS_BROXTON(i915))
1508                 bxt_whitelist_build(engine);
1509         else if (IS_SKYLAKE(i915))
1510                 skl_whitelist_build(engine);
1511         else if (INTEL_GEN(i915) <= 8)
1512                 return;
1513         else
1514                 MISSING_CASE(INTEL_GEN(i915));
1515
1516         wa_init_finish(w);
1517 }
1518
1519 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1520 {
1521         const struct i915_wa_list *wal = &engine->whitelist;
1522         struct intel_uncore *uncore = engine->uncore;
1523         const u32 base = engine->mmio_base;
1524         struct i915_wa *wa;
1525         unsigned int i;
1526
1527         if (!wal->count)
1528                 return;
1529
1530         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1531                 intel_uncore_write(uncore,
1532                                    RING_FORCE_TO_NONPRIV(base, i),
1533                                    i915_mmio_reg_offset(wa->reg));
1534
1535         /* And clear the rest just in case of garbage */
1536         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1537                 intel_uncore_write(uncore,
1538                                    RING_FORCE_TO_NONPRIV(base, i),
1539                                    i915_mmio_reg_offset(RING_NOPID(base)));
1540 }
1541
1542 static void
1543 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1544 {
1545         struct drm_i915_private *i915 = engine->i915;
1546
1547         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1548                 /*
1549                  * Wa_1607138336:tgl
1550                  * Wa_1607063988:tgl
1551                  */
1552                 wa_write_or(wal,
1553                             GEN9_CTX_PREEMPT_REG,
1554                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1555
1556                 /*
1557                  * Wa_1607030317:tgl
1558                  * Wa_1607186500:tgl
1559                  * Wa_1607297627:tgl there is 3 entries for this WA on BSpec, 2
1560                  * of then says it is fixed on B0 the other one says it is
1561                  * permanent
1562                  */
1563                 wa_masked_en(wal,
1564                              GEN6_RC_SLEEP_PSMI_CONTROL,
1565                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1566                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1567
1568                 /*
1569                  * Wa_1606679103:tgl
1570                  * (see also Wa_1606682166:icl)
1571                  */
1572                 wa_write_or(wal,
1573                             GEN7_SARCHKMD,
1574                             GEN7_DISABLE_SAMPLER_PREFETCH);
1575
1576                 /* Wa_1407928979:tgl */
1577                 wa_write_or(wal,
1578                             GEN7_FF_THREAD_MODE,
1579                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1580
1581                 /* Wa_1408615072:tgl */
1582                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1583                             VSUNIT_CLKGATE_DIS_TGL);
1584         }
1585
1586         if (IS_TIGERLAKE(i915)) {
1587                 /* Wa_1606931601:tgl */
1588                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1589
1590                 /* Wa_1409804808:tgl */
1591                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1592                              GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1593
1594                 /* Wa_1606700617:tgl */
1595                 wa_masked_en(wal,
1596                              GEN9_CS_DEBUG_MODE1,
1597                              FF_DOP_CLOCK_GATE_DISABLE);
1598
1599                 /*
1600                  * Wa_1409085225:tgl
1601                  * Wa_14010229206:tgl
1602                  */
1603                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1604         }
1605
1606         if (IS_GEN(i915, 11)) {
1607                 /* This is not an Wa. Enable for better image quality */
1608                 wa_masked_en(wal,
1609                              _3D_CHICKEN3,
1610                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1611
1612                 /* WaPipelineFlushCoherentLines:icl */
1613                 wa_write_or(wal,
1614                             GEN8_L3SQCREG4,
1615                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1616
1617                 /*
1618                  * Wa_1405543622:icl
1619                  * Formerly known as WaGAPZPriorityScheme
1620                  */
1621                 wa_write_or(wal,
1622                             GEN8_GARBCNTL,
1623                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1624
1625                 /*
1626                  * Wa_1604223664:icl
1627                  * Formerly known as WaL3BankAddressHashing
1628                  */
1629                 wa_write_masked_or(wal,
1630                                    GEN8_GARBCNTL,
1631                                    GEN11_HASH_CTRL_EXCL_MASK,
1632                                    GEN11_HASH_CTRL_EXCL_BIT0);
1633                 wa_write_masked_or(wal,
1634                                    GEN11_GLBLINVL,
1635                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1636                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1637
1638                 /*
1639                  * Wa_1405733216:icl
1640                  * Formerly known as WaDisableCleanEvicts
1641                  */
1642                 wa_write_or(wal,
1643                             GEN8_L3SQCREG4,
1644                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1645
1646                 /* WaForwardProgressSoftReset:icl */
1647                 wa_write_or(wal,
1648                             GEN10_SCRATCH_LNCF2,
1649                             PMFLUSHDONE_LNICRSDROP |
1650                             PMFLUSH_GAPL3UNBLOCK |
1651                             PMFLUSHDONE_LNEBLK);
1652
1653                 /* Wa_1406609255:icl (pre-prod) */
1654                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1655                         wa_write_or(wal,
1656                                     GEN7_SARCHKMD,
1657                                     GEN7_DISABLE_DEMAND_PREFETCH);
1658
1659                 /* Wa_1606682166:icl */
1660                 wa_write_or(wal,
1661                             GEN7_SARCHKMD,
1662                             GEN7_DISABLE_SAMPLER_PREFETCH);
1663
1664                 /* Wa_1409178092:icl */
1665                 wa_write_masked_or(wal,
1666                                    GEN11_SCRATCH2,
1667                                    GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1668                                    0);
1669
1670                 /* WaEnable32PlaneMode:icl */
1671                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1672                              GEN11_ENABLE_32_PLANE_MODE);
1673
1674                 /*
1675                  * Wa_1408615072:icl,ehl  (vsunit)
1676                  * Wa_1407596294:icl,ehl  (hsunit)
1677                  */
1678                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1679                             VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1680
1681                 /* Wa_1407352427:icl,ehl */
1682                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1683                             PSDUNIT_CLKGATE_DIS);
1684
1685                 /* Wa_1406680159:icl,ehl */
1686                 wa_write_or(wal,
1687                             SUBSLICE_UNIT_LEVEL_CLKGATE,
1688                             GWUNIT_CLKGATE_DIS);
1689
1690                 /*
1691                  * Wa_1408767742:icl[a2..forever],ehl[all]
1692                  * Wa_1605460711:icl[a0..c0]
1693                  */
1694                 wa_write_or(wal,
1695                             GEN7_FF_THREAD_MODE,
1696                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1697         }
1698
1699         if (IS_GEN_RANGE(i915, 9, 12)) {
1700                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1701                 wa_masked_en(wal,
1702                              GEN7_FF_SLICE_CS_CHICKEN1,
1703                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1704         }
1705
1706         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1707                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1708                 wa_write_or(wal,
1709                             GEN8_GARBCNTL,
1710                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1711         }
1712
1713         if (IS_BROXTON(i915)) {
1714                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1715                 wa_masked_en(wal,
1716                              FF_SLICE_CS_CHICKEN2,
1717                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1718         }
1719
1720         if (IS_GEN(i915, 9)) {
1721                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1722                 wa_masked_en(wal,
1723                              GEN9_CSFE_CHICKEN1_RCS,
1724                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1725
1726                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1727                 wa_write_or(wal,
1728                             BDW_SCRATCH1,
1729                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1730
1731                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1732                 if (IS_GEN9_LP(i915))
1733                         wa_write_masked_or(wal,
1734                                            GEN8_L3SQCREG1,
1735                                            L3_PRIO_CREDITS_MASK,
1736                                            L3_GENERAL_PRIO_CREDITS(62) |
1737                                            L3_HIGH_PRIO_CREDITS(2));
1738
1739                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1740                 wa_write_or(wal,
1741                             GEN8_L3SQCREG4,
1742                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1743         }
1744
1745         if (IS_GEN(i915, 7))
1746                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1747                 wa_masked_en(wal,
1748                              GFX_MODE_GEN7,
1749                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1750
1751         if (IS_GEN_RANGE(i915, 6, 7))
1752                 /*
1753                  * We need to disable the AsyncFlip performance optimisations in
1754                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
1755                  * already be programmed to '1' on all products.
1756                  *
1757                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1758                  */
1759                 wa_masked_en(wal,
1760                              MI_MODE,
1761                              ASYNC_FLIP_PERF_DISABLE);
1762
1763         if (IS_GEN(i915, 6)) {
1764                 /*
1765                  * Required for the hardware to program scanline values for
1766                  * waiting
1767                  * WaEnableFlushTlbInvalidationMode:snb
1768                  */
1769                 wa_masked_en(wal,
1770                              GFX_MODE,
1771                              GFX_TLB_INVALIDATE_EXPLICIT);
1772
1773                 /*
1774                  * From the Sandybridge PRM, volume 1 part 3, page 24:
1775                  * "If this bit is set, STCunit will have LRA as replacement
1776                  *  policy. [...] This bit must be reset. LRA replacement
1777                  *  policy is not supported."
1778                  */
1779                 wa_masked_dis(wal,
1780                               CACHE_MODE_0,
1781                               CM0_STC_EVICT_DISABLE_LRA_SNB);
1782         }
1783
1784         if (IS_GEN_RANGE(i915, 4, 6))
1785                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1786                 wa_add(wal, MI_MODE,
1787                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
1788                        /* XXX bit doesn't stick on Broadwater */
1789                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
1790 }
1791
1792 static void
1793 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1794 {
1795         struct drm_i915_private *i915 = engine->i915;
1796
1797         /* WaKBLVECSSemaphoreWaitPoll:kbl */
1798         if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1799                 wa_write(wal,
1800                          RING_SEMA_WAIT_POLL(engine->mmio_base),
1801                          1);
1802         }
1803 }
1804
1805 static void
1806 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1807 {
1808         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
1809                 return;
1810
1811         if (engine->class == RENDER_CLASS)
1812                 rcs_engine_wa_init(engine, wal);
1813         else
1814                 xcs_engine_wa_init(engine, wal);
1815 }
1816
1817 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1818 {
1819         struct i915_wa_list *wal = &engine->wa_list;
1820
1821         if (INTEL_GEN(engine->i915) < 4)
1822                 return;
1823
1824         wa_init_start(wal, "engine", engine->name);
1825         engine_init_workarounds(engine, wal);
1826         wa_init_finish(wal);
1827 }
1828
1829 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1830 {
1831         wa_list_apply(engine->uncore, &engine->wa_list);
1832 }
1833
1834 static struct i915_vma *
1835 create_scratch(struct i915_address_space *vm, int count)
1836 {
1837         struct drm_i915_gem_object *obj;
1838         struct i915_vma *vma;
1839         unsigned int size;
1840         int err;
1841
1842         size = round_up(count * sizeof(u32), PAGE_SIZE);
1843         obj = i915_gem_object_create_internal(vm->i915, size);
1844         if (IS_ERR(obj))
1845                 return ERR_CAST(obj);
1846
1847         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1848
1849         vma = i915_vma_instance(obj, vm, NULL);
1850         if (IS_ERR(vma)) {
1851                 err = PTR_ERR(vma);
1852                 goto err_obj;
1853         }
1854
1855         err = i915_vma_pin(vma, 0, 0,
1856                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
1857         if (err)
1858                 goto err_obj;
1859
1860         return vma;
1861
1862 err_obj:
1863         i915_gem_object_put(obj);
1864         return ERR_PTR(err);
1865 }
1866
1867 static const struct {
1868         u32 start;
1869         u32 end;
1870 } mcr_ranges_gen8[] = {
1871         { .start = 0x5500, .end = 0x55ff },
1872         { .start = 0x7000, .end = 0x7fff },
1873         { .start = 0x9400, .end = 0x97ff },
1874         { .start = 0xb000, .end = 0xb3ff },
1875         { .start = 0xe000, .end = 0xe7ff },
1876         {},
1877 };
1878
1879 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
1880 {
1881         int i;
1882
1883         if (INTEL_GEN(i915) < 8)
1884                 return false;
1885
1886         /*
1887          * Registers in these ranges are affected by the MCR selector
1888          * which only controls CPU initiated MMIO. Routing does not
1889          * work for CS access so we cannot verify them on this path.
1890          */
1891         for (i = 0; mcr_ranges_gen8[i].start; i++)
1892                 if (offset >= mcr_ranges_gen8[i].start &&
1893                     offset <= mcr_ranges_gen8[i].end)
1894                         return true;
1895
1896         return false;
1897 }
1898
1899 static int
1900 wa_list_srm(struct i915_request *rq,
1901             const struct i915_wa_list *wal,
1902             struct i915_vma *vma)
1903 {
1904         struct drm_i915_private *i915 = rq->i915;
1905         unsigned int i, count = 0;
1906         const struct i915_wa *wa;
1907         u32 srm, *cs;
1908
1909         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1910         if (INTEL_GEN(i915) >= 8)
1911                 srm++;
1912
1913         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1914                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
1915                         count++;
1916         }
1917
1918         cs = intel_ring_begin(rq, 4 * count);
1919         if (IS_ERR(cs))
1920                 return PTR_ERR(cs);
1921
1922         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1923                 u32 offset = i915_mmio_reg_offset(wa->reg);
1924
1925                 if (mcr_range(i915, offset))
1926                         continue;
1927
1928                 *cs++ = srm;
1929                 *cs++ = offset;
1930                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
1931                 *cs++ = 0;
1932         }
1933         intel_ring_advance(rq, cs);
1934
1935         return 0;
1936 }
1937
1938 static int engine_wa_list_verify(struct intel_context *ce,
1939                                  const struct i915_wa_list * const wal,
1940                                  const char *from)
1941 {
1942         const struct i915_wa *wa;
1943         struct i915_request *rq;
1944         struct i915_vma *vma;
1945         unsigned int i;
1946         u32 *results;
1947         int err;
1948
1949         if (!wal->count)
1950                 return 0;
1951
1952         vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
1953         if (IS_ERR(vma))
1954                 return PTR_ERR(vma);
1955
1956         intel_engine_pm_get(ce->engine);
1957         rq = intel_context_create_request(ce);
1958         intel_engine_pm_put(ce->engine);
1959         if (IS_ERR(rq)) {
1960                 err = PTR_ERR(rq);
1961                 goto err_vma;
1962         }
1963
1964         i915_vma_lock(vma);
1965         err = i915_request_await_object(rq, vma->obj, true);
1966         if (err == 0)
1967                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1968         i915_vma_unlock(vma);
1969         if (err) {
1970                 i915_request_add(rq);
1971                 goto err_vma;
1972         }
1973
1974         err = wa_list_srm(rq, wal, vma);
1975         if (err)
1976                 goto err_vma;
1977
1978         i915_request_get(rq);
1979         i915_request_add(rq);
1980         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1981                 err = -ETIME;
1982                 goto err_rq;
1983         }
1984
1985         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1986         if (IS_ERR(results)) {
1987                 err = PTR_ERR(results);
1988                 goto err_rq;
1989         }
1990
1991         err = 0;
1992         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1993                 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg)))
1994                         continue;
1995
1996                 if (!wa_verify(wa, results[i], wal->name, from))
1997                         err = -ENXIO;
1998         }
1999
2000         i915_gem_object_unpin_map(vma->obj);
2001
2002 err_rq:
2003         i915_request_put(rq);
2004 err_vma:
2005         i915_vma_unpin(vma);
2006         i915_vma_put(vma);
2007         return err;
2008 }
2009
2010 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2011                                     const char *from)
2012 {
2013         return engine_wa_list_verify(engine->kernel_context,
2014                                      &engine->wa_list,
2015                                      from);
2016 }
2017
2018 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2019 #include "selftest_workarounds.c"
2020 #endif