2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2018 Intel Corporation
8 #include "intel_context.h"
9 #include "intel_engine_pm.h"
11 #include "intel_ring.h"
12 #include "intel_workarounds.h"
15 * DOC: Hardware workarounds
17 * This file is intended as a central place to implement most [1]_ of the
18 * required workarounds for hardware to work as originally intended. They fall
19 * in five basic categories depending on how/when they are applied:
21 * - Workarounds that touch registers that are saved/restored to/from the HW
22 * context image. The list is emitted (via Load Register Immediate commands)
23 * everytime a new context is created.
24 * - GT workarounds. The list of these WAs is applied whenever these registers
25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26 * - Display workarounds. The list is applied during display clock-gating
28 * - Workarounds that whitelist a privileged register, so that UMDs can manage
29 * them directly. This is just a special case of a MMMIO workaround (as we
30 * write the list of these to/be-whitelisted registers to some special HW
32 * - Workaround batchbuffers, that get executed automatically by the hardware
33 * on every HW context restore.
35 * .. [1] Please notice that there are other WAs that, due to their nature,
36 * cannot be applied from a central place. Those are peppered around the rest
37 * of the code, as needed.
39 * .. [2] Technically, some registers are powercontext saved & restored, so they
40 * survive a suspend/resume. In practice, writing them again is not too
41 * costly and simplifies things. We can revisit this in the future.
46 * Keep things in this file ordered by WA type, as per the above (context, GT,
47 * display, register whitelist, batchbuffer). Then, inside each type, keep the
50 * - Infrastructure functions and macros
51 * - WAs per platform in standard gen/chrono order
52 * - Public functions to init or apply the given workaround type.
55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
58 wal->engine_name = engine_name;
61 #define WA_LIST_CHUNK (1 << 4)
63 static void wa_init_finish(struct i915_wa_list *wal)
65 /* Trim unused entries. */
66 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
67 struct i915_wa *list = kmemdup(wal->list,
68 wal->count * sizeof(*list),
80 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
81 wal->wa_count, wal->name, wal->engine_name);
84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
86 unsigned int addr = i915_mmio_reg_offset(wa->reg);
87 unsigned int start = 0, end = wal->count;
88 const unsigned int grow = WA_LIST_CHUNK;
91 GEM_BUG_ON(!is_power_of_2(grow));
93 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
96 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
99 DRM_ERROR("No space for workaround init!\n");
104 memcpy(list, wal->list, sizeof(*wa) * wal->count);
109 while (start < end) {
110 unsigned int mid = start + (end - start) / 2;
112 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
114 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
117 wa_ = &wal->list[mid];
119 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
120 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
121 i915_mmio_reg_offset(wa_->reg),
124 wa_->set &= ~wa->clr;
130 wa_->read |= wa->read;
136 wa_ = &wal->list[wal->count++];
139 while (wa_-- > wal->list) {
140 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
141 i915_mmio_reg_offset(wa_[1].reg));
142 if (i915_mmio_reg_offset(wa_[1].reg) >
143 i915_mmio_reg_offset(wa_[0].reg))
146 swap(wa_[1], wa_[0]);
150 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
151 u32 clear, u32 set, u32 read_mask)
153 struct i915_wa wa = {
164 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
166 wa_add(wal, reg, clear, set, clear);
170 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
172 wa_write_masked_or(wal, reg, ~0, set);
176 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
178 wa_write_masked_or(wal, reg, set, set);
182 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
184 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
188 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
190 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
193 #define WA_SET_BIT_MASKED(addr, mask) \
194 wa_masked_en(wal, (addr), (mask))
196 #define WA_CLR_BIT_MASKED(addr, mask) \
197 wa_masked_dis(wal, (addr), (mask))
199 #define WA_SET_FIELD_MASKED(addr, mask, value) \
200 wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
202 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
203 struct i915_wa_list *wal)
205 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
207 /* WaDisableAsyncFlipPerfMode:bdw,chv */
208 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
210 /* WaDisablePartialInstShootdown:bdw,chv */
211 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
212 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
214 /* Use Force Non-Coherent whenever executing a 3D context. This is a
215 * workaround for for a possible hang in the unlikely event a TLB
216 * invalidation occurs during a PSD flush.
218 /* WaForceEnableNonCoherent:bdw,chv */
219 /* WaHdcDisableFetchWhenMasked:bdw,chv */
220 WA_SET_BIT_MASKED(HDC_CHICKEN0,
221 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
222 HDC_FORCE_NON_COHERENT);
224 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
225 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
226 * polygons in the same 8x4 pixel/sample area to be processed without
227 * stalling waiting for the earlier ones to write to Hierarchical Z
230 * This optimization is off by default for BDW and CHV; turn it on.
232 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
234 /* Wa4x4STCOptimizationDisable:bdw,chv */
235 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
238 * BSpec recommends 8x4 when MSAA is used,
239 * however in practice 16x4 seems fastest.
241 * Note that PS/WM thread counts depend on the WIZ hashing
242 * disable bit, which we don't touch here, but it's good
243 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
245 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
246 GEN6_WIZ_HASHING_MASK,
247 GEN6_WIZ_HASHING_16x4);
250 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
251 struct i915_wa_list *wal)
253 struct drm_i915_private *i915 = engine->i915;
255 gen8_ctx_workarounds_init(engine, wal);
257 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
258 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
260 /* WaDisableDopClockGating:bdw
262 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
263 * to disable EUTC clock gating.
265 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
266 DOP_CLOCK_GATING_DISABLE);
268 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
269 GEN8_SAMPLER_POWER_BYPASS_DIS);
271 WA_SET_BIT_MASKED(HDC_CHICKEN0,
272 /* WaForceContextSaveRestoreNonCoherent:bdw */
273 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
274 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
275 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
278 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
279 struct i915_wa_list *wal)
281 gen8_ctx_workarounds_init(engine, wal);
283 /* WaDisableThreadStallDopClockGating:chv */
284 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
286 /* Improve HiZ throughput on CHV. */
287 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
290 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
291 struct i915_wa_list *wal)
293 struct drm_i915_private *i915 = engine->i915;
296 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
298 * Must match Display Engine. See
299 * WaCompressedResourceDisplayNewHashMode.
301 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
302 GEN9_PBE_COMPRESSED_HASH_SELECTION);
303 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
304 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
307 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
308 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
309 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
310 FLOW_CONTROL_ENABLE |
311 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
313 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
314 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
315 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
316 GEN9_ENABLE_YV12_BUGFIX |
317 GEN9_ENABLE_GPGPU_PREEMPTION);
319 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
320 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
321 WA_SET_BIT_MASKED(CACHE_MODE_1,
322 GEN8_4x4_STC_OPTIMIZATION_DISABLE |
323 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
325 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
326 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
327 GEN9_CCS_TLB_PREFETCH_ENABLE);
329 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
330 WA_SET_BIT_MASKED(HDC_CHICKEN0,
331 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
332 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
334 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
335 * both tied to WaForceContextSaveRestoreNonCoherent
336 * in some hsds for skl. We keep the tie for all gen9. The
337 * documentation is a bit hazy and so we want to get common behaviour,
338 * even though there is no clear evidence we would need both on kbl/bxt.
339 * This area has been source of system hangs so we play it safe
340 * and mimic the skl regardless of what bspec says.
342 * Use Force Non-Coherent whenever executing a 3D context. This
343 * is a workaround for a possible hang in the unlikely event
344 * a TLB invalidation occurs during a PSD flush.
347 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
348 WA_SET_BIT_MASKED(HDC_CHICKEN0,
349 HDC_FORCE_NON_COHERENT);
351 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
352 if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
353 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
354 GEN8_SAMPLER_POWER_BYPASS_DIS);
356 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
357 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
360 * Supporting preemption with fine-granularity requires changes in the
361 * batch buffer programming. Since we can't break old userspace, we
362 * need to set our default preemption level to safe value. Userspace is
363 * still able to use more fine-grained preemption levels, since in
364 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
365 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
366 * not real HW workarounds, but merely a way to start using preemption
367 * while maintaining old contract with userspace.
370 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
371 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
373 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
374 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
375 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
376 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
378 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
379 if (IS_GEN9_LP(i915))
380 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
383 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
384 struct i915_wa_list *wal)
386 struct drm_i915_private *i915 = engine->i915;
387 u8 vals[3] = { 0, 0, 0 };
390 for (i = 0; i < 3; i++) {
394 * Only consider slices where one, and only one, subslice has 7
397 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
401 * subslice_7eu[i] != 0 (because of the check above) and
402 * ss_max == 4 (maximum number of subslices possible per slice)
406 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
410 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
413 /* Tune IZ hashing. See intel_device_info_runtime_init() */
414 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
415 GEN9_IZ_HASHING_MASK(2) |
416 GEN9_IZ_HASHING_MASK(1) |
417 GEN9_IZ_HASHING_MASK(0),
418 GEN9_IZ_HASHING(2, vals[2]) |
419 GEN9_IZ_HASHING(1, vals[1]) |
420 GEN9_IZ_HASHING(0, vals[0]));
423 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
424 struct i915_wa_list *wal)
426 gen9_ctx_workarounds_init(engine, wal);
427 skl_tune_iz_hashing(engine, wal);
430 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
431 struct i915_wa_list *wal)
433 gen9_ctx_workarounds_init(engine, wal);
435 /* WaDisableThreadStallDopClockGating:bxt */
436 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
437 STALL_DOP_GATING_DISABLE);
439 /* WaToEnableHwFixForPushConstHWBug:bxt */
440 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
441 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
444 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
445 struct i915_wa_list *wal)
447 struct drm_i915_private *i915 = engine->i915;
449 gen9_ctx_workarounds_init(engine, wal);
451 /* WaToEnableHwFixForPushConstHWBug:kbl */
452 if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
453 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
454 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
456 /* WaDisableSbeCacheDispatchPortSharing:kbl */
457 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
458 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
461 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
462 struct i915_wa_list *wal)
464 gen9_ctx_workarounds_init(engine, wal);
466 /* WaToEnableHwFixForPushConstHWBug:glk */
467 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
468 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
471 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
472 struct i915_wa_list *wal)
474 gen9_ctx_workarounds_init(engine, wal);
476 /* WaToEnableHwFixForPushConstHWBug:cfl */
477 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
478 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
480 /* WaDisableSbeCacheDispatchPortSharing:cfl */
481 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
482 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
485 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
486 struct i915_wa_list *wal)
488 struct drm_i915_private *i915 = engine->i915;
490 /* WaForceContextSaveRestoreNonCoherent:cnl */
491 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
492 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
494 /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
495 if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
496 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
498 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
499 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
500 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
502 /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
503 if (IS_CNL_REVID(i915, 0, CNL_REVID_B0))
504 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
505 GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
507 /* WaPushConstantDereferenceHoldDisable:cnl */
508 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
510 /* FtrEnableFastAnisoL1BankingFix:cnl */
511 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
513 /* WaDisable3DMidCmdPreemption:cnl */
514 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
516 /* WaDisableGPGPUMidCmdPreemption:cnl */
517 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
518 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
519 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
521 /* WaDisableEarlyEOT:cnl */
522 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
525 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
526 struct i915_wa_list *wal)
528 struct drm_i915_private *i915 = engine->i915;
530 /* WaDisableBankHangMode:icl */
533 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
536 /* Wa_1604370585:icl (pre-prod)
537 * Formerly known as WaPushConstantDereferenceHoldDisable
539 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
540 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
541 PUSH_CONSTANT_DEREF_DISABLE);
543 /* WaForceEnableNonCoherent:icl
544 * This is not the same workaround as in early Gen9 platforms, where
545 * lacking this could cause system hangs, but coherency performance
546 * overhead is high and only a few compute workloads really need it
547 * (the register is whitelisted in hardware now, so UMDs can opt in
548 * for coherency if they have a good reason).
550 WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
552 /* Wa_2006611047:icl (pre-prod)
553 * Formerly known as WaDisableImprovedTdlClkGating
555 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
556 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
557 GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
559 /* Wa_2006665173:icl (pre-prod) */
560 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
561 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
562 GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
564 /* WaEnableFloatBlendOptimization:icl */
565 wa_write_masked_or(wal,
567 0, /* write-only, so skip validation */
568 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
570 /* WaDisableGPGPUMidThreadPreemption:icl */
571 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
572 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
573 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
575 /* allow headerless messages for preemptible GPGPU context */
576 WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
577 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
580 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
581 struct i915_wa_list *wal)
583 /* Wa_1409142259:tgl */
584 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
585 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
588 * Wa_1604555607:gen12 and Wa_1608008084:gen12
589 * FF_MODE2 register will return the wrong value when read. The default
590 * value for this register is zero for all fields and there are no bit
591 * masks. So instead of doing a RMW we should just write the TDS timer
592 * value for Wa_1604555607.
594 wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK,
595 FF_MODE2_TDS_TIMER_128, 0);
599 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
600 struct i915_wa_list *wal,
603 struct drm_i915_private *i915 = engine->i915;
605 if (engine->class != RENDER_CLASS)
608 wa_init_start(wal, name, engine->name);
610 if (IS_GEN(i915, 12))
611 tgl_ctx_workarounds_init(engine, wal);
612 else if (IS_GEN(i915, 11))
613 icl_ctx_workarounds_init(engine, wal);
614 else if (IS_CANNONLAKE(i915))
615 cnl_ctx_workarounds_init(engine, wal);
616 else if (IS_COFFEELAKE(i915))
617 cfl_ctx_workarounds_init(engine, wal);
618 else if (IS_GEMINILAKE(i915))
619 glk_ctx_workarounds_init(engine, wal);
620 else if (IS_KABYLAKE(i915))
621 kbl_ctx_workarounds_init(engine, wal);
622 else if (IS_BROXTON(i915))
623 bxt_ctx_workarounds_init(engine, wal);
624 else if (IS_SKYLAKE(i915))
625 skl_ctx_workarounds_init(engine, wal);
626 else if (IS_CHERRYVIEW(i915))
627 chv_ctx_workarounds_init(engine, wal);
628 else if (IS_BROADWELL(i915))
629 bdw_ctx_workarounds_init(engine, wal);
630 else if (INTEL_GEN(i915) < 8)
633 MISSING_CASE(INTEL_GEN(i915));
638 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
640 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
643 int intel_engine_emit_ctx_wa(struct i915_request *rq)
645 struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
654 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
658 cs = intel_ring_begin(rq, (wal->count * 2 + 2));
662 *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
663 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
664 *cs++ = i915_mmio_reg_offset(wa->reg);
669 intel_ring_advance(rq, cs);
671 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
679 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
681 /* WaDisableKillLogic:bxt,skl,kbl */
682 if (!IS_COFFEELAKE(i915))
688 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
690 * Must match Display Engine. See
691 * WaCompressedResourceDisplayNewHashMode.
695 MMCD_PCLA | MMCD_HOTSPOT_EN);
698 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
701 BDW_DISABLE_HDC_INVALIDATION);
705 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
707 gen9_gt_workarounds_init(i915, wal);
709 /* WaDisableGafsUnitClkGating:skl */
712 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
714 /* WaInPlaceDecompressionHang:skl */
715 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
717 GEN9_GAMT_ECO_REG_RW_IA,
718 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
722 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
724 gen9_gt_workarounds_init(i915, wal);
726 /* WaInPlaceDecompressionHang:bxt */
728 GEN9_GAMT_ECO_REG_RW_IA,
729 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
733 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
735 gen9_gt_workarounds_init(i915, wal);
737 /* WaDisableDynamicCreditSharing:kbl */
738 if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
741 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
743 /* WaDisableGafsUnitClkGating:kbl */
746 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
748 /* WaInPlaceDecompressionHang:kbl */
750 GEN9_GAMT_ECO_REG_RW_IA,
751 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
755 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
757 gen9_gt_workarounds_init(i915, wal);
761 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
763 gen9_gt_workarounds_init(i915, wal);
765 /* WaDisableGafsUnitClkGating:cfl */
768 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
770 /* WaInPlaceDecompressionHang:cfl */
772 GEN9_GAMT_ECO_REG_RW_IA,
773 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
777 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
779 const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
780 unsigned int slice, subslice;
781 u32 l3_en, mcr, mcr_mask;
783 GEM_BUG_ON(INTEL_GEN(i915) < 10);
786 * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
787 * L3Banks could be fused off in single slice scenario. If that is
788 * the case, we might need to program MCR select to a valid L3Bank
789 * by default, to make sure we correctly read certain registers
790 * later on (in the range 0xB100 - 0xB3FF).
792 * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
793 * Before any MMIO read into slice/subslice specific registers, MCR
794 * packet control register needs to be programmed to point to any
795 * enabled s/ss pair. Otherwise, incorrect values will be returned.
796 * This means each subsequent MMIO read will be forwarded to an
797 * specific s/ss combination, but this is OK since these registers
798 * are consistent across s/ss in almost all cases. In the rare
799 * occasions, such as INSTDONE, where this value is dependent
800 * on s/ss combo, the read should be done with read_subslice_reg.
802 * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
803 * to which subslice, or to which L3 bank, the respective mmio reads
804 * will go, we have to find a common index which works for both
807 * Case where we cannot find a common index fortunately should not
808 * happen in production hardware, so we only emit a warning instead of
809 * implementing something more complex that requires checking the range
810 * of every MMIO read.
813 if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
815 intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
818 DRM_DEBUG_DRIVER("L3 fuse = %x\n", l3_fuse);
819 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
824 slice = fls(sseu->slice_mask) - 1;
825 subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
827 DRM_WARN("No common index found between subslice mask %x and L3 bank mask %x!\n",
828 intel_sseu_get_subslices(sseu, slice), l3_en);
829 subslice = fls(l3_en);
830 drm_WARN_ON(&i915->drm, !subslice);
834 if (INTEL_GEN(i915) >= 11) {
835 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
836 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
838 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
839 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
842 DRM_DEBUG_DRIVER("MCR slice/subslice = %x\n", mcr);
844 wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
848 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
850 wa_init_mcr(i915, wal);
852 /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
853 if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
856 GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
858 /* WaInPlaceDecompressionHang:cnl */
860 GEN9_GAMT_ECO_REG_RW_IA,
861 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
865 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
867 wa_init_mcr(i915, wal);
869 /* WaInPlaceDecompressionHang:icl */
871 GEN9_GAMT_ECO_REG_RW_IA,
872 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
874 /* WaModifyGamTlbPartitioning:icl */
875 wa_write_masked_or(wal,
876 GEN11_GACB_PERF_CTRL,
877 GEN11_HASH_CTRL_MASK,
878 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
881 * Formerly known as WaCL2SFHalfMaxAlloc
885 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
886 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
889 * Formerly known as WaDisCtxReload
892 GEN8_GAMW_ECO_DEV_RW_IA,
893 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
895 /* Wa_1405779004:icl (pre-prod) */
896 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
898 SLICE_UNIT_LEVEL_CLKGATE,
899 MSCUNIT_CLKGATE_DIS);
901 /* Wa_1406680159:icl */
903 SUBSLICE_UNIT_LEVEL_CLKGATE,
906 /* Wa_1406838659:icl (pre-prod) */
907 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
909 INF_UNIT_LEVEL_CLKGATE,
913 * Formerly known as WaGamTlbPendError
917 GAMT_CHKN_DISABLE_L3_COH_PIPE);
919 /* Wa_1607087056:icl */
921 SLICE_UNIT_LEVEL_CLKGATE,
922 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
926 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
928 /* Wa_1409420604:tgl */
929 if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
931 SUBSLICE_UNIT_LEVEL_CLKGATE2,
932 CPSSUNIT_CLKGATE_DIS);
934 /* Wa_1409180338:tgl */
935 if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
937 SLICE_UNIT_LEVEL_CLKGATE,
938 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
942 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
944 if (IS_GEN(i915, 12))
945 tgl_gt_workarounds_init(i915, wal);
946 else if (IS_GEN(i915, 11))
947 icl_gt_workarounds_init(i915, wal);
948 else if (IS_CANNONLAKE(i915))
949 cnl_gt_workarounds_init(i915, wal);
950 else if (IS_COFFEELAKE(i915))
951 cfl_gt_workarounds_init(i915, wal);
952 else if (IS_GEMINILAKE(i915))
953 glk_gt_workarounds_init(i915, wal);
954 else if (IS_KABYLAKE(i915))
955 kbl_gt_workarounds_init(i915, wal);
956 else if (IS_BROXTON(i915))
957 bxt_gt_workarounds_init(i915, wal);
958 else if (IS_SKYLAKE(i915))
959 skl_gt_workarounds_init(i915, wal);
960 else if (INTEL_GEN(i915) <= 8)
963 MISSING_CASE(INTEL_GEN(i915));
966 void intel_gt_init_workarounds(struct drm_i915_private *i915)
968 struct i915_wa_list *wal = &i915->gt_wa_list;
970 wa_init_start(wal, "GT", "global");
971 gt_init_workarounds(i915, wal);
975 static enum forcewake_domains
976 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
978 enum forcewake_domains fw = 0;
982 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
983 fw |= intel_uncore_forcewake_for_reg(uncore,
992 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
994 if ((cur ^ wa->set) & wa->read) {
995 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
996 name, from, i915_mmio_reg_offset(wa->reg),
997 cur, cur & wa->read, wa->set);
1006 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1008 enum forcewake_domains fw;
1009 unsigned long flags;
1016 fw = wal_get_fw_for_rmw(uncore, wal);
1018 spin_lock_irqsave(&uncore->lock, flags);
1019 intel_uncore_forcewake_get__locked(uncore, fw);
1021 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1023 intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1025 intel_uncore_write_fw(uncore, wa->reg, wa->set);
1026 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1028 intel_uncore_read_fw(uncore, wa->reg),
1029 wal->name, "application");
1032 intel_uncore_forcewake_put__locked(uncore, fw);
1033 spin_unlock_irqrestore(&uncore->lock, flags);
1036 void intel_gt_apply_workarounds(struct intel_gt *gt)
1038 wa_list_apply(gt->uncore, >->i915->gt_wa_list);
1041 static bool wa_list_verify(struct intel_uncore *uncore,
1042 const struct i915_wa_list *wal,
1049 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1051 intel_uncore_read(uncore, wa->reg),
1057 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1059 return wa_list_verify(gt->uncore, >->i915->gt_wa_list, from);
1062 static inline bool is_nonpriv_flags_valid(u32 flags)
1064 /* Check only valid flag bits are set */
1065 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1068 /* NB: Only 3 out of 4 enum values are valid for access field */
1069 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1070 RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1077 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1079 struct i915_wa wa = {
1083 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1086 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1089 wa.reg.reg |= flags;
1094 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1096 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1099 static void gen9_whitelist_build(struct i915_wa_list *w)
1101 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1102 whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1104 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1105 whitelist_reg(w, GEN8_CS_CHICKEN1);
1107 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1108 whitelist_reg(w, GEN8_HDC_CHICKEN1);
1110 /* WaSendPushConstantsFromMMIO:skl,bxt */
1111 whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1114 static void skl_whitelist_build(struct intel_engine_cs *engine)
1116 struct i915_wa_list *w = &engine->whitelist;
1118 if (engine->class != RENDER_CLASS)
1121 gen9_whitelist_build(w);
1123 /* WaDisableLSQCROPERFforOCL:skl */
1124 whitelist_reg(w, GEN8_L3SQCREG4);
1127 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1129 if (engine->class != RENDER_CLASS)
1132 gen9_whitelist_build(&engine->whitelist);
1135 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1137 struct i915_wa_list *w = &engine->whitelist;
1139 if (engine->class != RENDER_CLASS)
1142 gen9_whitelist_build(w);
1144 /* WaDisableLSQCROPERFforOCL:kbl */
1145 whitelist_reg(w, GEN8_L3SQCREG4);
1148 static void glk_whitelist_build(struct intel_engine_cs *engine)
1150 struct i915_wa_list *w = &engine->whitelist;
1152 if (engine->class != RENDER_CLASS)
1155 gen9_whitelist_build(w);
1157 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1158 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1161 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1163 struct i915_wa_list *w = &engine->whitelist;
1165 if (engine->class != RENDER_CLASS)
1168 gen9_whitelist_build(w);
1171 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1173 * This covers 4 register which are next to one another :
1174 * - PS_INVOCATION_COUNT
1175 * - PS_INVOCATION_COUNT_UDW
1177 * - PS_DEPTH_COUNT_UDW
1179 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1180 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1181 RING_FORCE_TO_NONPRIV_RANGE_4);
1184 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1186 struct i915_wa_list *w = &engine->whitelist;
1188 if (engine->class != RENDER_CLASS)
1191 /* WaEnablePreemptionGranularityControlByUMD:cnl */
1192 whitelist_reg(w, GEN8_CS_CHICKEN1);
1195 static void icl_whitelist_build(struct intel_engine_cs *engine)
1197 struct i915_wa_list *w = &engine->whitelist;
1199 switch (engine->class) {
1201 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1202 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1204 /* WaAllowUMDToModifySamplerMode:icl */
1205 whitelist_reg(w, GEN10_SAMPLER_MODE);
1207 /* WaEnableStateCacheRedirectToCS:icl */
1208 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1211 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1213 * This covers 4 register which are next to one another :
1214 * - PS_INVOCATION_COUNT
1215 * - PS_INVOCATION_COUNT_UDW
1217 * - PS_DEPTH_COUNT_UDW
1219 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1220 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1221 RING_FORCE_TO_NONPRIV_RANGE_4);
1224 case VIDEO_DECODE_CLASS:
1225 /* hucStatusRegOffset */
1226 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1227 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1228 /* hucUKernelHdrInfoRegOffset */
1229 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1230 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1231 /* hucStatus2RegOffset */
1232 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1233 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1241 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1243 struct i915_wa_list *w = &engine->whitelist;
1245 switch (engine->class) {
1248 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1250 * This covers 4 registers which are next to one another :
1251 * - PS_INVOCATION_COUNT
1252 * - PS_INVOCATION_COUNT_UDW
1254 * - PS_DEPTH_COUNT_UDW
1256 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1257 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1258 RING_FORCE_TO_NONPRIV_RANGE_4);
1260 /* Wa_1808121037:tgl */
1261 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1263 /* Wa_1806527549:tgl */
1264 whitelist_reg(w, HIZ_CHICKEN);
1271 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1273 struct drm_i915_private *i915 = engine->i915;
1274 struct i915_wa_list *w = &engine->whitelist;
1276 wa_init_start(w, "whitelist", engine->name);
1278 if (IS_GEN(i915, 12))
1279 tgl_whitelist_build(engine);
1280 else if (IS_GEN(i915, 11))
1281 icl_whitelist_build(engine);
1282 else if (IS_CANNONLAKE(i915))
1283 cnl_whitelist_build(engine);
1284 else if (IS_COFFEELAKE(i915))
1285 cfl_whitelist_build(engine);
1286 else if (IS_GEMINILAKE(i915))
1287 glk_whitelist_build(engine);
1288 else if (IS_KABYLAKE(i915))
1289 kbl_whitelist_build(engine);
1290 else if (IS_BROXTON(i915))
1291 bxt_whitelist_build(engine);
1292 else if (IS_SKYLAKE(i915))
1293 skl_whitelist_build(engine);
1294 else if (INTEL_GEN(i915) <= 8)
1297 MISSING_CASE(INTEL_GEN(i915));
1302 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1304 const struct i915_wa_list *wal = &engine->whitelist;
1305 struct intel_uncore *uncore = engine->uncore;
1306 const u32 base = engine->mmio_base;
1313 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1314 intel_uncore_write(uncore,
1315 RING_FORCE_TO_NONPRIV(base, i),
1316 i915_mmio_reg_offset(wa->reg));
1318 /* And clear the rest just in case of garbage */
1319 for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1320 intel_uncore_write(uncore,
1321 RING_FORCE_TO_NONPRIV(base, i),
1322 i915_mmio_reg_offset(RING_NOPID(base)));
1326 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1328 struct drm_i915_private *i915 = engine->i915;
1330 if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1331 /* Wa_1606700617:tgl */
1333 GEN9_CS_DEBUG_MODE1,
1334 FF_DOP_CLOCK_GATE_DISABLE);
1341 GEN9_CTX_PREEMPT_REG,
1342 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1347 * Wa_1607297627:tgl there is 3 entries for this WA on BSpec, 2
1348 * of then says it is fixed on B0 the other one says it is
1352 GEN6_RC_SLEEP_PSMI_CONTROL,
1353 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1354 GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1358 * (see also Wa_1606682166:icl)
1362 GEN7_DISABLE_SAMPLER_PREFETCH);
1364 /* Wa_1407928979:tgl */
1366 GEN7_FF_THREAD_MODE,
1367 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1371 * Wa_14010229206:tgl
1373 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1376 if (IS_TIGERLAKE(i915)) {
1377 /* Wa_1606931601:tgl */
1378 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1380 /* Wa_1409804808:tgl */
1381 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1382 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1385 if (IS_GEN(i915, 11)) {
1386 /* This is not an Wa. Enable for better image quality */
1389 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1391 /* WaPipelineFlushCoherentLines:icl */
1394 GEN8_LQSC_FLUSH_COHERENT_LINES);
1398 * Formerly known as WaGAPZPriorityScheme
1402 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1406 * Formerly known as WaL3BankAddressHashing
1408 wa_write_masked_or(wal,
1410 GEN11_HASH_CTRL_EXCL_MASK,
1411 GEN11_HASH_CTRL_EXCL_BIT0);
1412 wa_write_masked_or(wal,
1414 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1415 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1419 * Formerly known as WaDisableCleanEvicts
1423 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1425 /* WaForwardProgressSoftReset:icl */
1427 GEN10_SCRATCH_LNCF2,
1428 PMFLUSHDONE_LNICRSDROP |
1429 PMFLUSH_GAPL3UNBLOCK |
1430 PMFLUSHDONE_LNEBLK);
1432 /* Wa_1406609255:icl (pre-prod) */
1433 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1436 GEN7_DISABLE_DEMAND_PREFETCH);
1438 /* Wa_1606682166:icl */
1441 GEN7_DISABLE_SAMPLER_PREFETCH);
1443 /* Wa_1409178092:icl */
1444 wa_write_masked_or(wal,
1446 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1450 if (IS_GEN_RANGE(i915, 9, 11)) {
1451 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl */
1453 GEN7_FF_SLICE_CS_CHICKEN1,
1454 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1457 if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1458 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1461 GEN9_GAPS_TSV_CREDIT_DISABLE);
1464 if (IS_BROXTON(i915)) {
1465 /* WaDisablePooledEuLoadBalancingFix:bxt */
1467 FF_SLICE_CS_CHICKEN2,
1468 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1471 if (IS_GEN(i915, 9)) {
1472 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1474 GEN9_CSFE_CHICKEN1_RCS,
1475 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1477 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1480 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1482 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1483 if (IS_GEN9_LP(i915))
1484 wa_write_masked_or(wal,
1486 L3_PRIO_CREDITS_MASK,
1487 L3_GENERAL_PRIO_CREDITS(62) |
1488 L3_HIGH_PRIO_CREDITS(2));
1490 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1493 GEN8_LQSC_FLUSH_COHERENT_LINES);
1496 if (IS_GEN(i915, 7))
1497 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1500 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1502 if (IS_GEN_RANGE(i915, 6, 7))
1504 * We need to disable the AsyncFlip performance optimisations in
1505 * order to use MI_WAIT_FOR_EVENT within the CS. It should
1506 * already be programmed to '1' on all products.
1508 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1512 ASYNC_FLIP_PERF_DISABLE);
1514 if (IS_GEN(i915, 6)) {
1516 * Required for the hardware to program scanline values for
1518 * WaEnableFlushTlbInvalidationMode:snb
1522 GFX_TLB_INVALIDATE_EXPLICIT);
1525 * From the Sandybridge PRM, volume 1 part 3, page 24:
1526 * "If this bit is set, STCunit will have LRA as replacement
1527 * policy. [...] This bit must be reset. LRA replacement
1528 * policy is not supported."
1532 CM0_STC_EVICT_DISABLE_LRA_SNB);
1535 if (IS_GEN_RANGE(i915, 4, 6))
1536 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1537 wa_add(wal, MI_MODE,
1538 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
1539 /* XXX bit doesn't stick on Broadwater */
1540 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
1544 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1546 struct drm_i915_private *i915 = engine->i915;
1548 /* WaKBLVECSSemaphoreWaitPoll:kbl */
1549 if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1551 RING_SEMA_WAIT_POLL(engine->mmio_base),
1557 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1559 if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
1562 if (engine->class == RENDER_CLASS)
1563 rcs_engine_wa_init(engine, wal);
1565 xcs_engine_wa_init(engine, wal);
1568 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1570 struct i915_wa_list *wal = &engine->wa_list;
1572 if (INTEL_GEN(engine->i915) < 4)
1575 wa_init_start(wal, "engine", engine->name);
1576 engine_init_workarounds(engine, wal);
1577 wa_init_finish(wal);
1580 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1582 wa_list_apply(engine->uncore, &engine->wa_list);
1585 static struct i915_vma *
1586 create_scratch(struct i915_address_space *vm, int count)
1588 struct drm_i915_gem_object *obj;
1589 struct i915_vma *vma;
1593 size = round_up(count * sizeof(u32), PAGE_SIZE);
1594 obj = i915_gem_object_create_internal(vm->i915, size);
1596 return ERR_CAST(obj);
1598 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1600 vma = i915_vma_instance(obj, vm, NULL);
1606 err = i915_vma_pin(vma, 0, 0,
1607 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
1614 i915_gem_object_put(obj);
1615 return ERR_PTR(err);
1618 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
1621 * Registers in this range are affected by the MCR selector
1622 * which only controls CPU initiated MMIO. Routing does not
1623 * work for CS access so we cannot verify them on this path.
1625 if (INTEL_GEN(i915) >= 8 && (offset >= 0xb000 && offset <= 0xb4ff))
1632 wa_list_srm(struct i915_request *rq,
1633 const struct i915_wa_list *wal,
1634 struct i915_vma *vma)
1636 struct drm_i915_private *i915 = rq->i915;
1637 unsigned int i, count = 0;
1638 const struct i915_wa *wa;
1641 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1642 if (INTEL_GEN(i915) >= 8)
1645 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1646 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
1650 cs = intel_ring_begin(rq, 4 * count);
1654 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1655 u32 offset = i915_mmio_reg_offset(wa->reg);
1657 if (mcr_range(i915, offset))
1662 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
1665 intel_ring_advance(rq, cs);
1670 static int engine_wa_list_verify(struct intel_context *ce,
1671 const struct i915_wa_list * const wal,
1674 const struct i915_wa *wa;
1675 struct i915_request *rq;
1676 struct i915_vma *vma;
1684 vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
1686 return PTR_ERR(vma);
1688 intel_engine_pm_get(ce->engine);
1689 rq = intel_context_create_request(ce);
1690 intel_engine_pm_put(ce->engine);
1697 err = i915_request_await_object(rq, vma->obj, true);
1699 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1700 i915_vma_unlock(vma);
1702 i915_request_add(rq);
1706 err = wa_list_srm(rq, wal, vma);
1710 i915_request_get(rq);
1711 i915_request_add(rq);
1712 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1717 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1718 if (IS_ERR(results)) {
1719 err = PTR_ERR(results);
1724 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1725 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg)))
1728 if (!wa_verify(wa, results[i], wal->name, from))
1732 i915_gem_object_unpin_map(vma->obj);
1735 i915_request_put(rq);
1737 i915_vma_unpin(vma);
1742 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
1745 return engine_wa_list_verify(engine->kernel_context,
1750 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1751 #include "selftest_workarounds.c"