2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
35 #include "display/intel_atomic.h"
36 #include "display/intel_bw.h"
37 #include "display/intel_display_types.h"
38 #include "display/intel_fbc.h"
39 #include "display/intel_sprite.h"
41 #include "gt/intel_llc.h"
44 #include "i915_fixed.h"
46 #include "i915_trace.h"
48 #include "intel_sideband.h"
49 #include "../../../platform/x86/intel_ips.h"
51 /* Stores plane specific WM parameters */
52 struct skl_wm_params {
53 bool x_tiled, y_tiled;
60 u32 plane_bytes_per_line;
61 uint_fixed_16_16_t plane_blocks_per_line;
62 uint_fixed_16_16_t y_tile_minimum;
67 /* used in computing the new watermarks state */
68 struct intel_wm_config {
69 unsigned int num_pipes_active;
74 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
76 if (HAS_LLC(dev_priv)) {
78 * WaCompressedResourceDisplayNewHashMode:skl,kbl
79 * Display WA #0390: skl,kbl
81 * Must match Sampler, Pixel Back End, and Media. See
82 * WaCompressedResourceSamplerPbeMediaNewHashMode.
84 I915_WRITE(CHICKEN_PAR1_1,
85 I915_READ(CHICKEN_PAR1_1) |
86 SKL_DE_COMPRESSED_HASH_MODE);
89 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
90 I915_WRITE(CHICKEN_PAR1_1,
91 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
93 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
94 I915_WRITE(GEN8_CHICKEN_DCPR_1,
95 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
98 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
99 * Display WA #0859: skl,bxt,kbl,glk,cfl
101 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
102 DISP_FBC_MEMORY_WAKE);
104 if (IS_SKYLAKE(dev_priv)) {
105 /* WaDisableDopClockGating */
106 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
107 & ~GEN7_DOP_CLOCK_GATE_ENABLE);
111 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
113 gen9_init_clock_gating(dev_priv);
115 /* WaDisableSDEUnitClockGating:bxt */
116 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
117 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
121 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
123 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
124 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
127 * Wa: Backlight PWM may stop in the asserted state, causing backlight
130 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
131 PWM1_GATING_DIS | PWM2_GATING_DIS);
134 * Lower the display internal timeout.
135 * This is needed to avoid any hard hangs when DSI port PLL
136 * is off and a MMIO access is attempted by any privilege
137 * application, using batch buffers or any other means.
139 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
142 * WaFbcTurnOffFbcWatermark:bxt
143 * Display WA #0562: bxt
145 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
149 * WaFbcHighMemBwCorruptionAvoidance:bxt
150 * Display WA #0883: bxt
152 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
153 ILK_DPFC_DISABLE_DUMMY0);
156 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
158 gen9_init_clock_gating(dev_priv);
161 * WaDisablePWMClockGating:glk
162 * Backlight PWM may stop in the asserted state, causing backlight
165 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
166 PWM1_GATING_DIS | PWM2_GATING_DIS);
169 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
173 tmp = I915_READ(CLKCFG);
175 switch (tmp & CLKCFG_FSB_MASK) {
177 dev_priv->fsb_freq = 533; /* 133*4 */
180 dev_priv->fsb_freq = 800; /* 200*4 */
183 dev_priv->fsb_freq = 667; /* 167*4 */
186 dev_priv->fsb_freq = 400; /* 100*4 */
190 switch (tmp & CLKCFG_MEM_MASK) {
192 dev_priv->mem_freq = 533;
195 dev_priv->mem_freq = 667;
198 dev_priv->mem_freq = 800;
202 /* detect pineview DDR3 setting */
203 tmp = I915_READ(CSHRDDR3CTL);
204 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
207 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
211 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
212 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
214 switch (ddrpll & 0xff) {
216 dev_priv->mem_freq = 800;
219 dev_priv->mem_freq = 1066;
222 dev_priv->mem_freq = 1333;
225 dev_priv->mem_freq = 1600;
228 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
230 dev_priv->mem_freq = 0;
234 switch (csipll & 0x3ff) {
236 dev_priv->fsb_freq = 3200;
239 dev_priv->fsb_freq = 3733;
242 dev_priv->fsb_freq = 4266;
245 dev_priv->fsb_freq = 4800;
248 dev_priv->fsb_freq = 5333;
251 dev_priv->fsb_freq = 5866;
254 dev_priv->fsb_freq = 6400;
257 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
259 dev_priv->fsb_freq = 0;
264 static const struct cxsr_latency cxsr_latency_table[] = {
265 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
266 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
267 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
268 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
269 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
271 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
272 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
273 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
274 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
275 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
277 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
278 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
279 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
280 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
281 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
283 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
284 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
285 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
286 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
287 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
289 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
290 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
291 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
292 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
293 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
295 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
296 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
297 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
298 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
299 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
302 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
307 const struct cxsr_latency *latency;
310 if (fsb == 0 || mem == 0)
313 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
314 latency = &cxsr_latency_table[i];
315 if (is_desktop == latency->is_desktop &&
316 is_ddr3 == latency->is_ddr3 &&
317 fsb == latency->fsb_freq && mem == latency->mem_freq)
321 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
326 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
330 vlv_punit_get(dev_priv);
332 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
334 val &= ~FORCE_DDR_HIGH_FREQ;
336 val |= FORCE_DDR_HIGH_FREQ;
337 val &= ~FORCE_DDR_LOW_FREQ;
338 val |= FORCE_DDR_FREQ_REQ_ACK;
339 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
341 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
342 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
343 drm_err(&dev_priv->drm,
344 "timed out waiting for Punit DDR DVFS request\n");
346 vlv_punit_put(dev_priv);
349 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
353 vlv_punit_get(dev_priv);
355 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
357 val |= DSP_MAXFIFO_PM5_ENABLE;
359 val &= ~DSP_MAXFIFO_PM5_ENABLE;
360 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
362 vlv_punit_put(dev_priv);
365 #define FW_WM(value, plane) \
366 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
368 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
373 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
374 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
375 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
376 POSTING_READ(FW_BLC_SELF_VLV);
377 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
378 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
379 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
380 POSTING_READ(FW_BLC_SELF);
381 } else if (IS_PINEVIEW(dev_priv)) {
382 val = I915_READ(DSPFW3);
383 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
385 val |= PINEVIEW_SELF_REFRESH_EN;
387 val &= ~PINEVIEW_SELF_REFRESH_EN;
388 I915_WRITE(DSPFW3, val);
389 POSTING_READ(DSPFW3);
390 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
391 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
392 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
393 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
394 I915_WRITE(FW_BLC_SELF, val);
395 POSTING_READ(FW_BLC_SELF);
396 } else if (IS_I915GM(dev_priv)) {
398 * FIXME can't find a bit like this for 915G, and
399 * and yet it does have the related watermark in
400 * FW_BLC_SELF. What's going on?
402 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
403 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
404 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
405 I915_WRITE(INSTPM, val);
406 POSTING_READ(INSTPM);
411 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
413 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
414 enableddisabled(enable),
415 enableddisabled(was_enabled));
421 * intel_set_memory_cxsr - Configure CxSR state
422 * @dev_priv: i915 device
423 * @enable: Allow vs. disallow CxSR
425 * Allow or disallow the system to enter a special CxSR
426 * (C-state self refresh) state. What typically happens in CxSR mode
427 * is that several display FIFOs may get combined into a single larger
428 * FIFO for a particular plane (so called max FIFO mode) to allow the
429 * system to defer memory fetches longer, and the memory will enter
432 * Note that enabling CxSR does not guarantee that the system enter
433 * this special mode, nor does it guarantee that the system stays
434 * in that mode once entered. So this just allows/disallows the system
435 * to autonomously utilize the CxSR mode. Other factors such as core
436 * C-states will affect when/if the system actually enters/exits the
439 * Note that on VLV/CHV this actually only controls the max FIFO mode,
440 * and the system is free to enter/exit memory self refresh at any time
441 * even when the use of CxSR has been disallowed.
443 * While the system is actually in the CxSR/max FIFO mode, some plane
444 * control registers will not get latched on vblank. Thus in order to
445 * guarantee the system will respond to changes in the plane registers
446 * we must always disallow CxSR prior to making changes to those registers.
447 * Unfortunately the system will re-evaluate the CxSR conditions at
448 * frame start which happens after vblank start (which is when the plane
449 * registers would get latched), so we can't proceed with the plane update
450 * during the same frame where we disallowed CxSR.
452 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
453 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
454 * the hardware w.r.t. HPLL SR when writing to plane registers.
455 * Disallowing just CxSR is sufficient.
457 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
461 mutex_lock(&dev_priv->wm.wm_mutex);
462 ret = _intel_set_memory_cxsr(dev_priv, enable);
463 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
464 dev_priv->wm.vlv.cxsr = enable;
465 else if (IS_G4X(dev_priv))
466 dev_priv->wm.g4x.cxsr = enable;
467 mutex_unlock(&dev_priv->wm.wm_mutex);
473 * Latency for FIFO fetches is dependent on several factors:
474 * - memory configuration (speed, channels)
476 * - current MCH state
477 * It can be fairly high in some situations, so here we assume a fairly
478 * pessimal value. It's a tradeoff between extra memory fetches (if we
479 * set this value too high, the FIFO will fetch frequently to stay full)
480 * and power consumption (set it too low to save power and we might see
481 * FIFO underruns and display "flicker").
483 * A value of 5us seems to be a good balance; safe for very low end
484 * platforms but not overly aggressive on lower latency configs.
486 static const int pessimal_latency_ns = 5000;
488 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
489 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
491 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
493 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
494 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
495 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
496 enum pipe pipe = crtc->pipe;
497 int sprite0_start, sprite1_start;
498 u32 dsparb, dsparb2, dsparb3;
502 dsparb = I915_READ(DSPARB);
503 dsparb2 = I915_READ(DSPARB2);
504 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
505 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
508 dsparb = I915_READ(DSPARB);
509 dsparb2 = I915_READ(DSPARB2);
510 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
511 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
514 dsparb2 = I915_READ(DSPARB2);
515 dsparb3 = I915_READ(DSPARB3);
516 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
517 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
524 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
525 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
526 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
527 fifo_state->plane[PLANE_CURSOR] = 63;
530 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
531 enum i9xx_plane_id i9xx_plane)
533 u32 dsparb = I915_READ(DSPARB);
536 size = dsparb & 0x7f;
537 if (i9xx_plane == PLANE_B)
538 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
540 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
541 dsparb, plane_name(i9xx_plane), size);
546 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
547 enum i9xx_plane_id i9xx_plane)
549 u32 dsparb = I915_READ(DSPARB);
552 size = dsparb & 0x1ff;
553 if (i9xx_plane == PLANE_B)
554 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
555 size >>= 1; /* Convert to cachelines */
557 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
558 dsparb, plane_name(i9xx_plane), size);
563 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
564 enum i9xx_plane_id i9xx_plane)
566 u32 dsparb = I915_READ(DSPARB);
569 size = dsparb & 0x7f;
570 size >>= 2; /* Convert to cachelines */
572 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
573 dsparb, plane_name(i9xx_plane), size);
578 /* Pineview has different values for various configs */
579 static const struct intel_watermark_params pnv_display_wm = {
580 .fifo_size = PINEVIEW_DISPLAY_FIFO,
581 .max_wm = PINEVIEW_MAX_WM,
582 .default_wm = PINEVIEW_DFT_WM,
583 .guard_size = PINEVIEW_GUARD_WM,
584 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
587 static const struct intel_watermark_params pnv_display_hplloff_wm = {
588 .fifo_size = PINEVIEW_DISPLAY_FIFO,
589 .max_wm = PINEVIEW_MAX_WM,
590 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
591 .guard_size = PINEVIEW_GUARD_WM,
592 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
595 static const struct intel_watermark_params pnv_cursor_wm = {
596 .fifo_size = PINEVIEW_CURSOR_FIFO,
597 .max_wm = PINEVIEW_CURSOR_MAX_WM,
598 .default_wm = PINEVIEW_CURSOR_DFT_WM,
599 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
600 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
603 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
604 .fifo_size = PINEVIEW_CURSOR_FIFO,
605 .max_wm = PINEVIEW_CURSOR_MAX_WM,
606 .default_wm = PINEVIEW_CURSOR_DFT_WM,
607 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
608 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
611 static const struct intel_watermark_params i965_cursor_wm_info = {
612 .fifo_size = I965_CURSOR_FIFO,
613 .max_wm = I965_CURSOR_MAX_WM,
614 .default_wm = I965_CURSOR_DFT_WM,
616 .cacheline_size = I915_FIFO_LINE_SIZE,
619 static const struct intel_watermark_params i945_wm_info = {
620 .fifo_size = I945_FIFO_SIZE,
621 .max_wm = I915_MAX_WM,
624 .cacheline_size = I915_FIFO_LINE_SIZE,
627 static const struct intel_watermark_params i915_wm_info = {
628 .fifo_size = I915_FIFO_SIZE,
629 .max_wm = I915_MAX_WM,
632 .cacheline_size = I915_FIFO_LINE_SIZE,
635 static const struct intel_watermark_params i830_a_wm_info = {
636 .fifo_size = I855GM_FIFO_SIZE,
637 .max_wm = I915_MAX_WM,
640 .cacheline_size = I830_FIFO_LINE_SIZE,
643 static const struct intel_watermark_params i830_bc_wm_info = {
644 .fifo_size = I855GM_FIFO_SIZE,
645 .max_wm = I915_MAX_WM/2,
648 .cacheline_size = I830_FIFO_LINE_SIZE,
651 static const struct intel_watermark_params i845_wm_info = {
652 .fifo_size = I830_FIFO_SIZE,
653 .max_wm = I915_MAX_WM,
656 .cacheline_size = I830_FIFO_LINE_SIZE,
660 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
661 * @pixel_rate: Pipe pixel rate in kHz
662 * @cpp: Plane bytes per pixel
663 * @latency: Memory wakeup latency in 0.1us units
665 * Compute the watermark using the method 1 or "small buffer"
666 * formula. The caller may additonally add extra cachelines
667 * to account for TLB misses and clock crossings.
669 * This method is concerned with the short term drain rate
670 * of the FIFO, ie. it does not account for blanking periods
671 * which would effectively reduce the average drain rate across
672 * a longer period. The name "small" refers to the fact the
673 * FIFO is relatively small compared to the amount of data
676 * The FIFO level vs. time graph might look something like:
680 * __---__---__ (- plane active, _ blanking)
683 * or perhaps like this:
686 * __----__----__ (- plane active, _ blanking)
690 * The watermark in bytes
692 static unsigned int intel_wm_method1(unsigned int pixel_rate,
694 unsigned int latency)
698 ret = mul_u32_u32(pixel_rate, cpp * latency);
699 ret = DIV_ROUND_UP_ULL(ret, 10000);
705 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
706 * @pixel_rate: Pipe pixel rate in kHz
707 * @htotal: Pipe horizontal total
708 * @width: Plane width in pixels
709 * @cpp: Plane bytes per pixel
710 * @latency: Memory wakeup latency in 0.1us units
712 * Compute the watermark using the method 2 or "large buffer"
713 * formula. The caller may additonally add extra cachelines
714 * to account for TLB misses and clock crossings.
716 * This method is concerned with the long term drain rate
717 * of the FIFO, ie. it does account for blanking periods
718 * which effectively reduce the average drain rate across
719 * a longer period. The name "large" refers to the fact the
720 * FIFO is relatively large compared to the amount of data
723 * The FIFO level vs. time graph might look something like:
728 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
732 * The watermark in bytes
734 static unsigned int intel_wm_method2(unsigned int pixel_rate,
738 unsigned int latency)
743 * FIXME remove once all users are computing
744 * watermarks in the correct place.
746 if (WARN_ON_ONCE(htotal == 0))
749 ret = (latency * pixel_rate) / (htotal * 10000);
750 ret = (ret + 1) * width * cpp;
756 * intel_calculate_wm - calculate watermark level
757 * @pixel_rate: pixel clock
758 * @wm: chip FIFO params
759 * @fifo_size: size of the FIFO buffer
760 * @cpp: bytes per pixel
761 * @latency_ns: memory latency for the platform
763 * Calculate the watermark level (the level at which the display plane will
764 * start fetching from memory again). Each chip has a different display
765 * FIFO size and allocation, so the caller needs to figure that out and pass
766 * in the correct intel_watermark_params structure.
768 * As the pixel clock runs, the FIFO will be drained at a rate that depends
769 * on the pixel size. When it reaches the watermark level, it'll start
770 * fetching FIFO line sized based chunks from memory until the FIFO fills
771 * past the watermark point. If the FIFO drains completely, a FIFO underrun
772 * will occur, and a display engine hang could result.
774 static unsigned int intel_calculate_wm(int pixel_rate,
775 const struct intel_watermark_params *wm,
776 int fifo_size, int cpp,
777 unsigned int latency_ns)
779 int entries, wm_size;
782 * Note: we need to make sure we don't overflow for various clock &
784 * clocks go from a few thousand to several hundred thousand.
785 * latency is usually a few thousand
787 entries = intel_wm_method1(pixel_rate, cpp,
789 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
791 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
793 wm_size = fifo_size - entries;
794 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
796 /* Don't promote wm_size to unsigned... */
797 if (wm_size > wm->max_wm)
798 wm_size = wm->max_wm;
800 wm_size = wm->default_wm;
803 * Bspec seems to indicate that the value shouldn't be lower than
804 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
805 * Lets go for 8 which is the burst size since certain platforms
806 * already use a hardcoded 8 (which is what the spec says should be
815 static bool is_disabling(int old, int new, int threshold)
817 return old >= threshold && new < threshold;
820 static bool is_enabling(int old, int new, int threshold)
822 return old < threshold && new >= threshold;
825 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
827 return dev_priv->wm.max_level + 1;
830 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
831 const struct intel_plane_state *plane_state)
833 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
835 /* FIXME check the 'enable' instead */
836 if (!crtc_state->hw.active)
840 * Treat cursor with fb as always visible since cursor updates
841 * can happen faster than the vrefresh rate, and the current
842 * watermark code doesn't handle that correctly. Cursor updates
843 * which set/clear the fb or change the cursor size are going
844 * to get throttled by intel_legacy_cursor_update() to work
845 * around this problem with the watermark code.
847 if (plane->id == PLANE_CURSOR)
848 return plane_state->hw.fb != NULL;
850 return plane_state->uapi.visible;
853 static bool intel_crtc_active(struct intel_crtc *crtc)
855 /* Be paranoid as we can arrive here with only partial
856 * state retrieved from the hardware during setup.
858 * We can ditch the adjusted_mode.crtc_clock check as soon
859 * as Haswell has gained clock readout/fastboot support.
861 * We can ditch the crtc->primary->state->fb check as soon as we can
862 * properly reconstruct framebuffers.
864 * FIXME: The intel_crtc->active here should be switched to
865 * crtc->state->active once we have proper CRTC states wired up
868 return crtc->active && crtc->base.primary->state->fb &&
869 crtc->config->hw.adjusted_mode.crtc_clock;
872 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
874 struct intel_crtc *crtc, *enabled = NULL;
876 for_each_intel_crtc(&dev_priv->drm, crtc) {
877 if (intel_crtc_active(crtc)) {
887 static void pnv_update_wm(struct intel_crtc *unused_crtc)
889 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
890 struct intel_crtc *crtc;
891 const struct cxsr_latency *latency;
895 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
900 drm_dbg_kms(&dev_priv->drm,
901 "Unknown FSB/MEM found, disable CxSR\n");
902 intel_set_memory_cxsr(dev_priv, false);
906 crtc = single_enabled_crtc(dev_priv);
908 const struct drm_display_mode *adjusted_mode =
909 &crtc->config->hw.adjusted_mode;
910 const struct drm_framebuffer *fb =
911 crtc->base.primary->state->fb;
912 int cpp = fb->format->cpp[0];
913 int clock = adjusted_mode->crtc_clock;
916 wm = intel_calculate_wm(clock, &pnv_display_wm,
917 pnv_display_wm.fifo_size,
918 cpp, latency->display_sr);
919 reg = I915_READ(DSPFW1);
920 reg &= ~DSPFW_SR_MASK;
921 reg |= FW_WM(wm, SR);
922 I915_WRITE(DSPFW1, reg);
923 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
926 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
927 pnv_display_wm.fifo_size,
928 4, latency->cursor_sr);
929 reg = I915_READ(DSPFW3);
930 reg &= ~DSPFW_CURSOR_SR_MASK;
931 reg |= FW_WM(wm, CURSOR_SR);
932 I915_WRITE(DSPFW3, reg);
934 /* Display HPLL off SR */
935 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
936 pnv_display_hplloff_wm.fifo_size,
937 cpp, latency->display_hpll_disable);
938 reg = I915_READ(DSPFW3);
939 reg &= ~DSPFW_HPLL_SR_MASK;
940 reg |= FW_WM(wm, HPLL_SR);
941 I915_WRITE(DSPFW3, reg);
943 /* cursor HPLL off SR */
944 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
945 pnv_display_hplloff_wm.fifo_size,
946 4, latency->cursor_hpll_disable);
947 reg = I915_READ(DSPFW3);
948 reg &= ~DSPFW_HPLL_CURSOR_MASK;
949 reg |= FW_WM(wm, HPLL_CURSOR);
950 I915_WRITE(DSPFW3, reg);
951 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
953 intel_set_memory_cxsr(dev_priv, true);
955 intel_set_memory_cxsr(dev_priv, false);
960 * Documentation says:
961 * "If the line size is small, the TLB fetches can get in the way of the
962 * data fetches, causing some lag in the pixel data return which is not
963 * accounted for in the above formulas. The following adjustment only
964 * needs to be applied if eight whole lines fit in the buffer at once.
965 * The WM is adjusted upwards by the difference between the FIFO size
966 * and the size of 8 whole lines. This adjustment is always performed
967 * in the actual pixel depth regardless of whether FBC is enabled or not."
969 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
971 int tlb_miss = fifo_size * 64 - width * cpp * 8;
973 return max(0, tlb_miss);
976 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
977 const struct g4x_wm_values *wm)
981 for_each_pipe(dev_priv, pipe)
982 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
985 FW_WM(wm->sr.plane, SR) |
986 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
987 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
988 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
990 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
991 FW_WM(wm->sr.fbc, FBC_SR) |
992 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
993 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
994 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
995 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
997 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
998 FW_WM(wm->sr.cursor, CURSOR_SR) |
999 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1000 FW_WM(wm->hpll.plane, HPLL_SR));
1002 POSTING_READ(DSPFW1);
1005 #define FW_WM_VLV(value, plane) \
1006 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1008 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1009 const struct vlv_wm_values *wm)
1013 for_each_pipe(dev_priv, pipe) {
1014 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1016 I915_WRITE(VLV_DDL(pipe),
1017 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1018 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1019 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1020 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1024 * Zero the (unused) WM1 watermarks, and also clear all the
1025 * high order bits so that there are no out of bounds values
1026 * present in the registers during the reprogramming.
1028 I915_WRITE(DSPHOWM, 0);
1029 I915_WRITE(DSPHOWM1, 0);
1030 I915_WRITE(DSPFW4, 0);
1031 I915_WRITE(DSPFW5, 0);
1032 I915_WRITE(DSPFW6, 0);
1035 FW_WM(wm->sr.plane, SR) |
1036 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1037 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1038 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1040 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1041 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1042 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1044 FW_WM(wm->sr.cursor, CURSOR_SR));
1046 if (IS_CHERRYVIEW(dev_priv)) {
1047 I915_WRITE(DSPFW7_CHV,
1048 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1049 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1050 I915_WRITE(DSPFW8_CHV,
1051 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1052 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1053 I915_WRITE(DSPFW9_CHV,
1054 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1055 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1057 FW_WM(wm->sr.plane >> 9, SR_HI) |
1058 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1059 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1060 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1061 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1062 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1063 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1064 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1065 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1066 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1069 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1070 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1072 FW_WM(wm->sr.plane >> 9, SR_HI) |
1073 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1074 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1075 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1076 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1077 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1078 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1081 POSTING_READ(DSPFW1);
1086 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1088 /* all latencies in usec */
1089 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1090 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1091 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1093 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1096 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1099 * DSPCNTR[13] supposedly controls whether the
1100 * primary plane can use the FIFO space otherwise
1101 * reserved for the sprite plane. It's not 100% clear
1102 * what the actual FIFO size is, but it looks like we
1103 * can happily set both primary and sprite watermarks
1104 * up to 127 cachelines. So that would seem to mean
1105 * that either DSPCNTR[13] doesn't do anything, or that
1106 * the total FIFO is >= 256 cachelines in size. Either
1107 * way, we don't seem to have to worry about this
1108 * repartitioning as the maximum watermark value the
1109 * register can hold for each plane is lower than the
1110 * minimum FIFO size.
1116 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1118 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1120 MISSING_CASE(plane_id);
1125 static int g4x_fbc_fifo_size(int level)
1128 case G4X_WM_LEVEL_SR:
1130 case G4X_WM_LEVEL_HPLL:
1133 MISSING_CASE(level);
1138 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1139 const struct intel_plane_state *plane_state,
1142 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1143 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1144 const struct drm_display_mode *adjusted_mode =
1145 &crtc_state->hw.adjusted_mode;
1146 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1147 unsigned int clock, htotal, cpp, width, wm;
1152 if (!intel_wm_plane_visible(crtc_state, plane_state))
1155 cpp = plane_state->hw.fb->format->cpp[0];
1158 * Not 100% sure which way ELK should go here as the
1159 * spec only says CL/CTG should assume 32bpp and BW
1160 * doesn't need to. But as these things followed the
1161 * mobile vs. desktop lines on gen3 as well, let's
1162 * assume ELK doesn't need this.
1164 * The spec also fails to list such a restriction for
1165 * the HPLL watermark, which seems a little strange.
1166 * Let's use 32bpp for the HPLL watermark as well.
1168 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1169 level != G4X_WM_LEVEL_NORMAL)
1172 clock = adjusted_mode->crtc_clock;
1173 htotal = adjusted_mode->crtc_htotal;
1175 width = drm_rect_width(&plane_state->uapi.dst);
1177 if (plane->id == PLANE_CURSOR) {
1178 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1179 } else if (plane->id == PLANE_PRIMARY &&
1180 level == G4X_WM_LEVEL_NORMAL) {
1181 wm = intel_wm_method1(clock, cpp, latency);
1183 unsigned int small, large;
1185 small = intel_wm_method1(clock, cpp, latency);
1186 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1188 wm = min(small, large);
1191 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1194 wm = DIV_ROUND_UP(wm, 64) + 2;
1196 return min_t(unsigned int, wm, USHRT_MAX);
1199 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1200 int level, enum plane_id plane_id, u16 value)
1202 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1205 for (; level < intel_wm_num_levels(dev_priv); level++) {
1206 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1208 dirty |= raw->plane[plane_id] != value;
1209 raw->plane[plane_id] = value;
1215 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1216 int level, u16 value)
1218 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1221 /* NORMAL level doesn't have an FBC watermark */
1222 level = max(level, G4X_WM_LEVEL_SR);
1224 for (; level < intel_wm_num_levels(dev_priv); level++) {
1225 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1227 dirty |= raw->fbc != value;
1234 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1235 const struct intel_plane_state *plane_state,
1238 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1239 const struct intel_plane_state *plane_state)
1241 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1242 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1243 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1244 enum plane_id plane_id = plane->id;
1248 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1249 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1250 if (plane_id == PLANE_PRIMARY)
1251 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1255 for (level = 0; level < num_levels; level++) {
1256 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1259 wm = g4x_compute_wm(crtc_state, plane_state, level);
1260 max_wm = g4x_plane_fifo_size(plane_id, level);
1265 dirty |= raw->plane[plane_id] != wm;
1266 raw->plane[plane_id] = wm;
1268 if (plane_id != PLANE_PRIMARY ||
1269 level == G4X_WM_LEVEL_NORMAL)
1272 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1273 raw->plane[plane_id]);
1274 max_wm = g4x_fbc_fifo_size(level);
1277 * FBC wm is not mandatory as we
1278 * can always just disable its use.
1283 dirty |= raw->fbc != wm;
1287 /* mark watermarks as invalid */
1288 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1290 if (plane_id == PLANE_PRIMARY)
1291 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1295 drm_dbg_kms(&dev_priv->drm,
1296 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1298 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1299 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1300 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1302 if (plane_id == PLANE_PRIMARY)
1303 drm_dbg_kms(&dev_priv->drm,
1304 "FBC watermarks: SR=%d, HPLL=%d\n",
1305 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1306 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1312 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1313 enum plane_id plane_id, int level)
1315 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1317 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1320 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1323 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1325 if (level > dev_priv->wm.max_level)
1328 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1329 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1330 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1333 /* mark all levels starting from 'level' as invalid */
1334 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1335 struct g4x_wm_state *wm_state, int level)
1337 if (level <= G4X_WM_LEVEL_NORMAL) {
1338 enum plane_id plane_id;
1340 for_each_plane_id_on_crtc(crtc, plane_id)
1341 wm_state->wm.plane[plane_id] = USHRT_MAX;
1344 if (level <= G4X_WM_LEVEL_SR) {
1345 wm_state->cxsr = false;
1346 wm_state->sr.cursor = USHRT_MAX;
1347 wm_state->sr.plane = USHRT_MAX;
1348 wm_state->sr.fbc = USHRT_MAX;
1351 if (level <= G4X_WM_LEVEL_HPLL) {
1352 wm_state->hpll_en = false;
1353 wm_state->hpll.cursor = USHRT_MAX;
1354 wm_state->hpll.plane = USHRT_MAX;
1355 wm_state->hpll.fbc = USHRT_MAX;
1359 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1362 if (level < G4X_WM_LEVEL_SR)
1365 if (level >= G4X_WM_LEVEL_SR &&
1366 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1369 if (level >= G4X_WM_LEVEL_HPLL &&
1370 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1376 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1378 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1379 struct intel_atomic_state *state =
1380 to_intel_atomic_state(crtc_state->uapi.state);
1381 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1382 int num_active_planes = hweight8(crtc_state->active_planes &
1383 ~BIT(PLANE_CURSOR));
1384 const struct g4x_pipe_wm *raw;
1385 const struct intel_plane_state *old_plane_state;
1386 const struct intel_plane_state *new_plane_state;
1387 struct intel_plane *plane;
1388 enum plane_id plane_id;
1390 unsigned int dirty = 0;
1392 for_each_oldnew_intel_plane_in_state(state, plane,
1394 new_plane_state, i) {
1395 if (new_plane_state->hw.crtc != &crtc->base &&
1396 old_plane_state->hw.crtc != &crtc->base)
1399 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1400 dirty |= BIT(plane->id);
1406 level = G4X_WM_LEVEL_NORMAL;
1407 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1410 raw = &crtc_state->wm.g4x.raw[level];
1411 for_each_plane_id_on_crtc(crtc, plane_id)
1412 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1414 level = G4X_WM_LEVEL_SR;
1415 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1418 raw = &crtc_state->wm.g4x.raw[level];
1419 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1420 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1421 wm_state->sr.fbc = raw->fbc;
1423 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1425 level = G4X_WM_LEVEL_HPLL;
1426 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1429 raw = &crtc_state->wm.g4x.raw[level];
1430 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1431 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1432 wm_state->hpll.fbc = raw->fbc;
1434 wm_state->hpll_en = wm_state->cxsr;
1439 if (level == G4X_WM_LEVEL_NORMAL)
1442 /* invalidate the higher levels */
1443 g4x_invalidate_wms(crtc, wm_state, level);
1446 * Determine if the FBC watermark(s) can be used. IF
1447 * this isn't the case we prefer to disable the FBC
1448 * watermark(s) rather than disable the SR/HPLL
1449 * level(s) entirely. 'level-1' is the highest valid
1452 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1457 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1459 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1460 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1461 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1462 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1463 struct intel_atomic_state *intel_state =
1464 to_intel_atomic_state(new_crtc_state->uapi.state);
1465 const struct intel_crtc_state *old_crtc_state =
1466 intel_atomic_get_old_crtc_state(intel_state, crtc);
1467 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1468 enum plane_id plane_id;
1470 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1471 *intermediate = *optimal;
1473 intermediate->cxsr = false;
1474 intermediate->hpll_en = false;
1478 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1479 !new_crtc_state->disable_cxsr;
1480 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1481 !new_crtc_state->disable_cxsr;
1482 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1484 for_each_plane_id_on_crtc(crtc, plane_id) {
1485 intermediate->wm.plane[plane_id] =
1486 max(optimal->wm.plane[plane_id],
1487 active->wm.plane[plane_id]);
1489 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1490 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1493 intermediate->sr.plane = max(optimal->sr.plane,
1495 intermediate->sr.cursor = max(optimal->sr.cursor,
1497 intermediate->sr.fbc = max(optimal->sr.fbc,
1500 intermediate->hpll.plane = max(optimal->hpll.plane,
1501 active->hpll.plane);
1502 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1503 active->hpll.cursor);
1504 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1507 drm_WARN_ON(&dev_priv->drm,
1508 (intermediate->sr.plane >
1509 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1510 intermediate->sr.cursor >
1511 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1512 intermediate->cxsr);
1513 drm_WARN_ON(&dev_priv->drm,
1514 (intermediate->sr.plane >
1515 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1516 intermediate->sr.cursor >
1517 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1518 intermediate->hpll_en);
1520 drm_WARN_ON(&dev_priv->drm,
1521 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1522 intermediate->fbc_en && intermediate->cxsr);
1523 drm_WARN_ON(&dev_priv->drm,
1524 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1525 intermediate->fbc_en && intermediate->hpll_en);
1529 * If our intermediate WM are identical to the final WM, then we can
1530 * omit the post-vblank programming; only update if it's different.
1532 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1533 new_crtc_state->wm.need_postvbl_update = true;
1538 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1539 struct g4x_wm_values *wm)
1541 struct intel_crtc *crtc;
1542 int num_active_pipes = 0;
1548 for_each_intel_crtc(&dev_priv->drm, crtc) {
1549 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1554 if (!wm_state->cxsr)
1556 if (!wm_state->hpll_en)
1557 wm->hpll_en = false;
1558 if (!wm_state->fbc_en)
1564 if (num_active_pipes != 1) {
1566 wm->hpll_en = false;
1570 for_each_intel_crtc(&dev_priv->drm, crtc) {
1571 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1572 enum pipe pipe = crtc->pipe;
1574 wm->pipe[pipe] = wm_state->wm;
1575 if (crtc->active && wm->cxsr)
1576 wm->sr = wm_state->sr;
1577 if (crtc->active && wm->hpll_en)
1578 wm->hpll = wm_state->hpll;
1582 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1584 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1585 struct g4x_wm_values new_wm = {};
1587 g4x_merge_wm(dev_priv, &new_wm);
1589 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1592 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1593 _intel_set_memory_cxsr(dev_priv, false);
1595 g4x_write_wm_values(dev_priv, &new_wm);
1597 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1598 _intel_set_memory_cxsr(dev_priv, true);
1603 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1604 struct intel_crtc *crtc)
1606 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1607 const struct intel_crtc_state *crtc_state =
1608 intel_atomic_get_new_crtc_state(state, crtc);
1610 mutex_lock(&dev_priv->wm.wm_mutex);
1611 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1612 g4x_program_watermarks(dev_priv);
1613 mutex_unlock(&dev_priv->wm.wm_mutex);
1616 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1617 struct intel_crtc *crtc)
1619 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1620 const struct intel_crtc_state *crtc_state =
1621 intel_atomic_get_new_crtc_state(state, crtc);
1623 if (!crtc_state->wm.need_postvbl_update)
1626 mutex_lock(&dev_priv->wm.wm_mutex);
1627 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1628 g4x_program_watermarks(dev_priv);
1629 mutex_unlock(&dev_priv->wm.wm_mutex);
1632 /* latency must be in 0.1us units. */
1633 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1634 unsigned int htotal,
1637 unsigned int latency)
1641 ret = intel_wm_method2(pixel_rate, htotal,
1642 width, cpp, latency);
1643 ret = DIV_ROUND_UP(ret, 64);
1648 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1650 /* all latencies in usec */
1651 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1653 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1655 if (IS_CHERRYVIEW(dev_priv)) {
1656 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1657 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1659 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1663 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1664 const struct intel_plane_state *plane_state,
1667 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1668 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1669 const struct drm_display_mode *adjusted_mode =
1670 &crtc_state->hw.adjusted_mode;
1671 unsigned int clock, htotal, cpp, width, wm;
1673 if (dev_priv->wm.pri_latency[level] == 0)
1676 if (!intel_wm_plane_visible(crtc_state, plane_state))
1679 cpp = plane_state->hw.fb->format->cpp[0];
1680 clock = adjusted_mode->crtc_clock;
1681 htotal = adjusted_mode->crtc_htotal;
1682 width = crtc_state->pipe_src_w;
1684 if (plane->id == PLANE_CURSOR) {
1686 * FIXME the formula gives values that are
1687 * too big for the cursor FIFO, and hence we
1688 * would never be able to use cursors. For
1689 * now just hardcode the watermark.
1693 wm = vlv_wm_method2(clock, htotal, width, cpp,
1694 dev_priv->wm.pri_latency[level] * 10);
1697 return min_t(unsigned int, wm, USHRT_MAX);
1700 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1702 return (active_planes & (BIT(PLANE_SPRITE0) |
1703 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1706 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1708 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1709 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1710 const struct g4x_pipe_wm *raw =
1711 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1712 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1713 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1714 int num_active_planes = hweight8(active_planes);
1715 const int fifo_size = 511;
1716 int fifo_extra, fifo_left = fifo_size;
1717 int sprite0_fifo_extra = 0;
1718 unsigned int total_rate;
1719 enum plane_id plane_id;
1722 * When enabling sprite0 after sprite1 has already been enabled
1723 * we tend to get an underrun unless sprite0 already has some
1724 * FIFO space allcoated. Hence we always allocate at least one
1725 * cacheline for sprite0 whenever sprite1 is enabled.
1727 * All other plane enable sequences appear immune to this problem.
1729 if (vlv_need_sprite0_fifo_workaround(active_planes))
1730 sprite0_fifo_extra = 1;
1732 total_rate = raw->plane[PLANE_PRIMARY] +
1733 raw->plane[PLANE_SPRITE0] +
1734 raw->plane[PLANE_SPRITE1] +
1737 if (total_rate > fifo_size)
1740 if (total_rate == 0)
1743 for_each_plane_id_on_crtc(crtc, plane_id) {
1746 if ((active_planes & BIT(plane_id)) == 0) {
1747 fifo_state->plane[plane_id] = 0;
1751 rate = raw->plane[plane_id];
1752 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1753 fifo_left -= fifo_state->plane[plane_id];
1756 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1757 fifo_left -= sprite0_fifo_extra;
1759 fifo_state->plane[PLANE_CURSOR] = 63;
1761 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1763 /* spread the remainder evenly */
1764 for_each_plane_id_on_crtc(crtc, plane_id) {
1770 if ((active_planes & BIT(plane_id)) == 0)
1773 plane_extra = min(fifo_extra, fifo_left);
1774 fifo_state->plane[plane_id] += plane_extra;
1775 fifo_left -= plane_extra;
1778 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1780 /* give it all to the first plane if none are active */
1781 if (active_planes == 0) {
1782 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1783 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1789 /* mark all levels starting from 'level' as invalid */
1790 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1791 struct vlv_wm_state *wm_state, int level)
1793 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1795 for (; level < intel_wm_num_levels(dev_priv); level++) {
1796 enum plane_id plane_id;
1798 for_each_plane_id_on_crtc(crtc, plane_id)
1799 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1801 wm_state->sr[level].cursor = USHRT_MAX;
1802 wm_state->sr[level].plane = USHRT_MAX;
1806 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1811 return fifo_size - wm;
1815 * Starting from 'level' set all higher
1816 * levels to 'value' in the "raw" watermarks.
1818 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1819 int level, enum plane_id plane_id, u16 value)
1821 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1822 int num_levels = intel_wm_num_levels(dev_priv);
1825 for (; level < num_levels; level++) {
1826 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1828 dirty |= raw->plane[plane_id] != value;
1829 raw->plane[plane_id] = value;
1835 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1836 const struct intel_plane_state *plane_state)
1838 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1839 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1840 enum plane_id plane_id = plane->id;
1841 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1845 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1846 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1850 for (level = 0; level < num_levels; level++) {
1851 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1852 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1853 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1858 dirty |= raw->plane[plane_id] != wm;
1859 raw->plane[plane_id] = wm;
1862 /* mark all higher levels as invalid */
1863 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1867 drm_dbg_kms(&dev_priv->drm,
1868 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1870 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1871 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1872 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1877 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1878 enum plane_id plane_id, int level)
1880 const struct g4x_pipe_wm *raw =
1881 &crtc_state->wm.vlv.raw[level];
1882 const struct vlv_fifo_state *fifo_state =
1883 &crtc_state->wm.vlv.fifo_state;
1885 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1888 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1890 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1891 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1892 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1893 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1896 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1898 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1899 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1900 struct intel_atomic_state *state =
1901 to_intel_atomic_state(crtc_state->uapi.state);
1902 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1903 const struct vlv_fifo_state *fifo_state =
1904 &crtc_state->wm.vlv.fifo_state;
1905 int num_active_planes = hweight8(crtc_state->active_planes &
1906 ~BIT(PLANE_CURSOR));
1907 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1908 const struct intel_plane_state *old_plane_state;
1909 const struct intel_plane_state *new_plane_state;
1910 struct intel_plane *plane;
1911 enum plane_id plane_id;
1913 unsigned int dirty = 0;
1915 for_each_oldnew_intel_plane_in_state(state, plane,
1917 new_plane_state, i) {
1918 if (new_plane_state->hw.crtc != &crtc->base &&
1919 old_plane_state->hw.crtc != &crtc->base)
1922 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1923 dirty |= BIT(plane->id);
1927 * DSPARB registers may have been reset due to the
1928 * power well being turned off. Make sure we restore
1929 * them to a consistent state even if no primary/sprite
1930 * planes are initially active.
1933 crtc_state->fifo_changed = true;
1938 /* cursor changes don't warrant a FIFO recompute */
1939 if (dirty & ~BIT(PLANE_CURSOR)) {
1940 const struct intel_crtc_state *old_crtc_state =
1941 intel_atomic_get_old_crtc_state(state, crtc);
1942 const struct vlv_fifo_state *old_fifo_state =
1943 &old_crtc_state->wm.vlv.fifo_state;
1945 ret = vlv_compute_fifo(crtc_state);
1949 if (needs_modeset ||
1950 memcmp(old_fifo_state, fifo_state,
1951 sizeof(*fifo_state)) != 0)
1952 crtc_state->fifo_changed = true;
1955 /* initially allow all levels */
1956 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1958 * Note that enabling cxsr with no primary/sprite planes
1959 * enabled can wedge the pipe. Hence we only allow cxsr
1960 * with exactly one enabled primary/sprite plane.
1962 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1964 for (level = 0; level < wm_state->num_levels; level++) {
1965 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1966 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1968 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1971 for_each_plane_id_on_crtc(crtc, plane_id) {
1972 wm_state->wm[level].plane[plane_id] =
1973 vlv_invert_wm_value(raw->plane[plane_id],
1974 fifo_state->plane[plane_id]);
1977 wm_state->sr[level].plane =
1978 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1979 raw->plane[PLANE_SPRITE0],
1980 raw->plane[PLANE_SPRITE1]),
1983 wm_state->sr[level].cursor =
1984 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1991 /* limit to only levels we can actually handle */
1992 wm_state->num_levels = level;
1994 /* invalidate the higher levels */
1995 vlv_invalidate_wms(crtc, wm_state, level);
2000 #define VLV_FIFO(plane, value) \
2001 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2003 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2004 struct intel_crtc *crtc)
2006 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2007 struct intel_uncore *uncore = &dev_priv->uncore;
2008 const struct intel_crtc_state *crtc_state =
2009 intel_atomic_get_new_crtc_state(state, crtc);
2010 const struct vlv_fifo_state *fifo_state =
2011 &crtc_state->wm.vlv.fifo_state;
2012 int sprite0_start, sprite1_start, fifo_size;
2013 u32 dsparb, dsparb2, dsparb3;
2015 if (!crtc_state->fifo_changed)
2018 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2019 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2020 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2022 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2023 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2025 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2028 * uncore.lock serves a double purpose here. It allows us to
2029 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2030 * it protects the DSPARB registers from getting clobbered by
2031 * parallel updates from multiple pipes.
2033 * intel_pipe_update_start() has already disabled interrupts
2034 * for us, so a plain spin_lock() is sufficient here.
2036 spin_lock(&uncore->lock);
2038 switch (crtc->pipe) {
2040 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2041 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2043 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2044 VLV_FIFO(SPRITEB, 0xff));
2045 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2046 VLV_FIFO(SPRITEB, sprite1_start));
2048 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2049 VLV_FIFO(SPRITEB_HI, 0x1));
2050 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2051 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2053 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2054 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2057 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2058 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2060 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2061 VLV_FIFO(SPRITED, 0xff));
2062 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2063 VLV_FIFO(SPRITED, sprite1_start));
2065 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2066 VLV_FIFO(SPRITED_HI, 0xff));
2067 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2068 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2070 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2071 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2074 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2075 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2077 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2078 VLV_FIFO(SPRITEF, 0xff));
2079 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2080 VLV_FIFO(SPRITEF, sprite1_start));
2082 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2083 VLV_FIFO(SPRITEF_HI, 0xff));
2084 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2085 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2087 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2088 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2094 intel_uncore_posting_read_fw(uncore, DSPARB);
2096 spin_unlock(&uncore->lock);
2101 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2103 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2104 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2105 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2106 struct intel_atomic_state *intel_state =
2107 to_intel_atomic_state(new_crtc_state->uapi.state);
2108 const struct intel_crtc_state *old_crtc_state =
2109 intel_atomic_get_old_crtc_state(intel_state, crtc);
2110 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2113 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2114 *intermediate = *optimal;
2116 intermediate->cxsr = false;
2120 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2121 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2122 !new_crtc_state->disable_cxsr;
2124 for (level = 0; level < intermediate->num_levels; level++) {
2125 enum plane_id plane_id;
2127 for_each_plane_id_on_crtc(crtc, plane_id) {
2128 intermediate->wm[level].plane[plane_id] =
2129 min(optimal->wm[level].plane[plane_id],
2130 active->wm[level].plane[plane_id]);
2133 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2134 active->sr[level].plane);
2135 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2136 active->sr[level].cursor);
2139 vlv_invalidate_wms(crtc, intermediate, level);
2143 * If our intermediate WM are identical to the final WM, then we can
2144 * omit the post-vblank programming; only update if it's different.
2146 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2147 new_crtc_state->wm.need_postvbl_update = true;
2152 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2153 struct vlv_wm_values *wm)
2155 struct intel_crtc *crtc;
2156 int num_active_pipes = 0;
2158 wm->level = dev_priv->wm.max_level;
2161 for_each_intel_crtc(&dev_priv->drm, crtc) {
2162 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2167 if (!wm_state->cxsr)
2171 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2174 if (num_active_pipes != 1)
2177 if (num_active_pipes > 1)
2178 wm->level = VLV_WM_LEVEL_PM2;
2180 for_each_intel_crtc(&dev_priv->drm, crtc) {
2181 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2182 enum pipe pipe = crtc->pipe;
2184 wm->pipe[pipe] = wm_state->wm[wm->level];
2185 if (crtc->active && wm->cxsr)
2186 wm->sr = wm_state->sr[wm->level];
2188 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2189 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2190 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2191 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2195 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2197 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2198 struct vlv_wm_values new_wm = {};
2200 vlv_merge_wm(dev_priv, &new_wm);
2202 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2205 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2206 chv_set_memory_dvfs(dev_priv, false);
2208 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2209 chv_set_memory_pm5(dev_priv, false);
2211 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2212 _intel_set_memory_cxsr(dev_priv, false);
2214 vlv_write_wm_values(dev_priv, &new_wm);
2216 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2217 _intel_set_memory_cxsr(dev_priv, true);
2219 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2220 chv_set_memory_pm5(dev_priv, true);
2222 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2223 chv_set_memory_dvfs(dev_priv, true);
2228 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2229 struct intel_crtc *crtc)
2231 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2232 const struct intel_crtc_state *crtc_state =
2233 intel_atomic_get_new_crtc_state(state, crtc);
2235 mutex_lock(&dev_priv->wm.wm_mutex);
2236 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2237 vlv_program_watermarks(dev_priv);
2238 mutex_unlock(&dev_priv->wm.wm_mutex);
2241 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2242 struct intel_crtc *crtc)
2244 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2245 const struct intel_crtc_state *crtc_state =
2246 intel_atomic_get_new_crtc_state(state, crtc);
2248 if (!crtc_state->wm.need_postvbl_update)
2251 mutex_lock(&dev_priv->wm.wm_mutex);
2252 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2253 vlv_program_watermarks(dev_priv);
2254 mutex_unlock(&dev_priv->wm.wm_mutex);
2257 static void i965_update_wm(struct intel_crtc *unused_crtc)
2259 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2260 struct intel_crtc *crtc;
2265 /* Calc sr entries for one plane configs */
2266 crtc = single_enabled_crtc(dev_priv);
2268 /* self-refresh has much higher latency */
2269 static const int sr_latency_ns = 12000;
2270 const struct drm_display_mode *adjusted_mode =
2271 &crtc->config->hw.adjusted_mode;
2272 const struct drm_framebuffer *fb =
2273 crtc->base.primary->state->fb;
2274 int clock = adjusted_mode->crtc_clock;
2275 int htotal = adjusted_mode->crtc_htotal;
2276 int hdisplay = crtc->config->pipe_src_w;
2277 int cpp = fb->format->cpp[0];
2280 entries = intel_wm_method2(clock, htotal,
2281 hdisplay, cpp, sr_latency_ns / 100);
2282 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2283 srwm = I965_FIFO_SIZE - entries;
2287 drm_dbg_kms(&dev_priv->drm,
2288 "self-refresh entries: %d, wm: %d\n",
2291 entries = intel_wm_method2(clock, htotal,
2292 crtc->base.cursor->state->crtc_w, 4,
2293 sr_latency_ns / 100);
2294 entries = DIV_ROUND_UP(entries,
2295 i965_cursor_wm_info.cacheline_size) +
2296 i965_cursor_wm_info.guard_size;
2298 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2299 if (cursor_sr > i965_cursor_wm_info.max_wm)
2300 cursor_sr = i965_cursor_wm_info.max_wm;
2302 drm_dbg_kms(&dev_priv->drm,
2303 "self-refresh watermark: display plane %d "
2304 "cursor %d\n", srwm, cursor_sr);
2306 cxsr_enabled = true;
2308 cxsr_enabled = false;
2309 /* Turn off self refresh if both pipes are enabled */
2310 intel_set_memory_cxsr(dev_priv, false);
2313 drm_dbg_kms(&dev_priv->drm,
2314 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2317 /* 965 has limitations... */
2318 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2322 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2323 FW_WM(8, PLANEC_OLD));
2324 /* update cursor SR watermark */
2325 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2328 intel_set_memory_cxsr(dev_priv, true);
2333 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2335 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2336 const struct intel_watermark_params *wm_info;
2341 int planea_wm, planeb_wm;
2342 struct intel_crtc *crtc, *enabled = NULL;
2344 if (IS_I945GM(dev_priv))
2345 wm_info = &i945_wm_info;
2346 else if (!IS_GEN(dev_priv, 2))
2347 wm_info = &i915_wm_info;
2349 wm_info = &i830_a_wm_info;
2351 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2352 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2353 if (intel_crtc_active(crtc)) {
2354 const struct drm_display_mode *adjusted_mode =
2355 &crtc->config->hw.adjusted_mode;
2356 const struct drm_framebuffer *fb =
2357 crtc->base.primary->state->fb;
2360 if (IS_GEN(dev_priv, 2))
2363 cpp = fb->format->cpp[0];
2365 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2366 wm_info, fifo_size, cpp,
2367 pessimal_latency_ns);
2370 planea_wm = fifo_size - wm_info->guard_size;
2371 if (planea_wm > (long)wm_info->max_wm)
2372 planea_wm = wm_info->max_wm;
2375 if (IS_GEN(dev_priv, 2))
2376 wm_info = &i830_bc_wm_info;
2378 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2379 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2380 if (intel_crtc_active(crtc)) {
2381 const struct drm_display_mode *adjusted_mode =
2382 &crtc->config->hw.adjusted_mode;
2383 const struct drm_framebuffer *fb =
2384 crtc->base.primary->state->fb;
2387 if (IS_GEN(dev_priv, 2))
2390 cpp = fb->format->cpp[0];
2392 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2393 wm_info, fifo_size, cpp,
2394 pessimal_latency_ns);
2395 if (enabled == NULL)
2400 planeb_wm = fifo_size - wm_info->guard_size;
2401 if (planeb_wm > (long)wm_info->max_wm)
2402 planeb_wm = wm_info->max_wm;
2405 drm_dbg_kms(&dev_priv->drm,
2406 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2408 if (IS_I915GM(dev_priv) && enabled) {
2409 struct drm_i915_gem_object *obj;
2411 obj = intel_fb_obj(enabled->base.primary->state->fb);
2413 /* self-refresh seems busted with untiled */
2414 if (!i915_gem_object_is_tiled(obj))
2419 * Overlay gets an aggressive default since video jitter is bad.
2423 /* Play safe and disable self-refresh before adjusting watermarks. */
2424 intel_set_memory_cxsr(dev_priv, false);
2426 /* Calc sr entries for one plane configs */
2427 if (HAS_FW_BLC(dev_priv) && enabled) {
2428 /* self-refresh has much higher latency */
2429 static const int sr_latency_ns = 6000;
2430 const struct drm_display_mode *adjusted_mode =
2431 &enabled->config->hw.adjusted_mode;
2432 const struct drm_framebuffer *fb =
2433 enabled->base.primary->state->fb;
2434 int clock = adjusted_mode->crtc_clock;
2435 int htotal = adjusted_mode->crtc_htotal;
2436 int hdisplay = enabled->config->pipe_src_w;
2440 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2443 cpp = fb->format->cpp[0];
2445 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2446 sr_latency_ns / 100);
2447 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2448 drm_dbg_kms(&dev_priv->drm,
2449 "self-refresh entries: %d\n", entries);
2450 srwm = wm_info->fifo_size - entries;
2454 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2455 I915_WRITE(FW_BLC_SELF,
2456 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2458 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2461 drm_dbg_kms(&dev_priv->drm,
2462 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2463 planea_wm, planeb_wm, cwm, srwm);
2465 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2466 fwater_hi = (cwm & 0x1f);
2468 /* Set request length to 8 cachelines per fetch */
2469 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2470 fwater_hi = fwater_hi | (1 << 8);
2472 I915_WRITE(FW_BLC, fwater_lo);
2473 I915_WRITE(FW_BLC2, fwater_hi);
2476 intel_set_memory_cxsr(dev_priv, true);
2479 static void i845_update_wm(struct intel_crtc *unused_crtc)
2481 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2482 struct intel_crtc *crtc;
2483 const struct drm_display_mode *adjusted_mode;
2487 crtc = single_enabled_crtc(dev_priv);
2491 adjusted_mode = &crtc->config->hw.adjusted_mode;
2492 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2494 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2495 4, pessimal_latency_ns);
2496 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2497 fwater_lo |= (3<<8) | planea_wm;
2499 drm_dbg_kms(&dev_priv->drm,
2500 "Setting FIFO watermarks - A: %d\n", planea_wm);
2502 I915_WRITE(FW_BLC, fwater_lo);
2505 /* latency must be in 0.1us units. */
2506 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2508 unsigned int latency)
2512 ret = intel_wm_method1(pixel_rate, cpp, latency);
2513 ret = DIV_ROUND_UP(ret, 64) + 2;
2518 /* latency must be in 0.1us units. */
2519 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2520 unsigned int htotal,
2523 unsigned int latency)
2527 ret = intel_wm_method2(pixel_rate, htotal,
2528 width, cpp, latency);
2529 ret = DIV_ROUND_UP(ret, 64) + 2;
2534 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2537 * Neither of these should be possible since this function shouldn't be
2538 * called if the CRTC is off or the plane is invisible. But let's be
2539 * extra paranoid to avoid a potential divide-by-zero if we screw up
2540 * elsewhere in the driver.
2544 if (WARN_ON(!horiz_pixels))
2547 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2550 struct ilk_wm_maximums {
2558 * For both WM_PIPE and WM_LP.
2559 * mem_value must be in 0.1us units.
2561 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2562 const struct intel_plane_state *plane_state,
2563 u32 mem_value, bool is_lp)
2565 u32 method1, method2;
2571 if (!intel_wm_plane_visible(crtc_state, plane_state))
2574 cpp = plane_state->hw.fb->format->cpp[0];
2576 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2581 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2582 crtc_state->hw.adjusted_mode.crtc_htotal,
2583 drm_rect_width(&plane_state->uapi.dst),
2586 return min(method1, method2);
2590 * For both WM_PIPE and WM_LP.
2591 * mem_value must be in 0.1us units.
2593 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2594 const struct intel_plane_state *plane_state,
2597 u32 method1, method2;
2603 if (!intel_wm_plane_visible(crtc_state, plane_state))
2606 cpp = plane_state->hw.fb->format->cpp[0];
2608 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2609 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2610 crtc_state->hw.adjusted_mode.crtc_htotal,
2611 drm_rect_width(&plane_state->uapi.dst),
2613 return min(method1, method2);
2617 * For both WM_PIPE and WM_LP.
2618 * mem_value must be in 0.1us units.
2620 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2621 const struct intel_plane_state *plane_state,
2629 if (!intel_wm_plane_visible(crtc_state, plane_state))
2632 cpp = plane_state->hw.fb->format->cpp[0];
2634 return ilk_wm_method2(crtc_state->pixel_rate,
2635 crtc_state->hw.adjusted_mode.crtc_htotal,
2636 drm_rect_width(&plane_state->uapi.dst),
2640 /* Only for WM_LP. */
2641 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2642 const struct intel_plane_state *plane_state,
2647 if (!intel_wm_plane_visible(crtc_state, plane_state))
2650 cpp = plane_state->hw.fb->format->cpp[0];
2652 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2657 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2659 if (INTEL_GEN(dev_priv) >= 8)
2661 else if (INTEL_GEN(dev_priv) >= 7)
2668 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2669 int level, bool is_sprite)
2671 if (INTEL_GEN(dev_priv) >= 8)
2672 /* BDW primary/sprite plane watermarks */
2673 return level == 0 ? 255 : 2047;
2674 else if (INTEL_GEN(dev_priv) >= 7)
2675 /* IVB/HSW primary/sprite plane watermarks */
2676 return level == 0 ? 127 : 1023;
2677 else if (!is_sprite)
2678 /* ILK/SNB primary plane watermarks */
2679 return level == 0 ? 127 : 511;
2681 /* ILK/SNB sprite plane watermarks */
2682 return level == 0 ? 63 : 255;
2686 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2688 if (INTEL_GEN(dev_priv) >= 7)
2689 return level == 0 ? 63 : 255;
2691 return level == 0 ? 31 : 63;
2694 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2696 if (INTEL_GEN(dev_priv) >= 8)
2702 /* Calculate the maximum primary/sprite plane watermark */
2703 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2705 const struct intel_wm_config *config,
2706 enum intel_ddb_partitioning ddb_partitioning,
2709 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2711 /* if sprites aren't enabled, sprites get nothing */
2712 if (is_sprite && !config->sprites_enabled)
2715 /* HSW allows LP1+ watermarks even with multiple pipes */
2716 if (level == 0 || config->num_pipes_active > 1) {
2717 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2720 * For some reason the non self refresh
2721 * FIFO size is only half of the self
2722 * refresh FIFO size on ILK/SNB.
2724 if (INTEL_GEN(dev_priv) <= 6)
2728 if (config->sprites_enabled) {
2729 /* level 0 is always calculated with 1:1 split */
2730 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2739 /* clamp to max that the registers can hold */
2740 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2743 /* Calculate the maximum cursor plane watermark */
2744 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2746 const struct intel_wm_config *config)
2748 /* HSW LP1+ watermarks w/ multiple pipes */
2749 if (level > 0 && config->num_pipes_active > 1)
2752 /* otherwise just report max that registers can hold */
2753 return ilk_cursor_wm_reg_max(dev_priv, level);
2756 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2758 const struct intel_wm_config *config,
2759 enum intel_ddb_partitioning ddb_partitioning,
2760 struct ilk_wm_maximums *max)
2762 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2763 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2764 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2765 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2768 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2770 struct ilk_wm_maximums *max)
2772 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2773 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2774 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2775 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2778 static bool ilk_validate_wm_level(int level,
2779 const struct ilk_wm_maximums *max,
2780 struct intel_wm_level *result)
2784 /* already determined to be invalid? */
2785 if (!result->enable)
2788 result->enable = result->pri_val <= max->pri &&
2789 result->spr_val <= max->spr &&
2790 result->cur_val <= max->cur;
2792 ret = result->enable;
2795 * HACK until we can pre-compute everything,
2796 * and thus fail gracefully if LP0 watermarks
2799 if (level == 0 && !result->enable) {
2800 if (result->pri_val > max->pri)
2801 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2802 level, result->pri_val, max->pri);
2803 if (result->spr_val > max->spr)
2804 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2805 level, result->spr_val, max->spr);
2806 if (result->cur_val > max->cur)
2807 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2808 level, result->cur_val, max->cur);
2810 result->pri_val = min_t(u32, result->pri_val, max->pri);
2811 result->spr_val = min_t(u32, result->spr_val, max->spr);
2812 result->cur_val = min_t(u32, result->cur_val, max->cur);
2813 result->enable = true;
2819 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2820 const struct intel_crtc *crtc,
2822 struct intel_crtc_state *crtc_state,
2823 const struct intel_plane_state *pristate,
2824 const struct intel_plane_state *sprstate,
2825 const struct intel_plane_state *curstate,
2826 struct intel_wm_level *result)
2828 u16 pri_latency = dev_priv->wm.pri_latency[level];
2829 u16 spr_latency = dev_priv->wm.spr_latency[level];
2830 u16 cur_latency = dev_priv->wm.cur_latency[level];
2832 /* WM1+ latency values stored in 0.5us units */
2840 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2841 pri_latency, level);
2842 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2846 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2849 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2851 result->enable = true;
2854 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2857 struct intel_uncore *uncore = &dev_priv->uncore;
2859 if (INTEL_GEN(dev_priv) >= 9) {
2862 int level, max_level = ilk_wm_max_level(dev_priv);
2864 /* read the first set of memory latencies[0:3] */
2865 val = 0; /* data0 to be programmed to 0 for first set */
2866 ret = sandybridge_pcode_read(dev_priv,
2867 GEN9_PCODE_READ_MEM_LATENCY,
2871 drm_err(&dev_priv->drm,
2872 "SKL Mailbox read error = %d\n", ret);
2876 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2877 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2878 GEN9_MEM_LATENCY_LEVEL_MASK;
2879 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2880 GEN9_MEM_LATENCY_LEVEL_MASK;
2881 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2882 GEN9_MEM_LATENCY_LEVEL_MASK;
2884 /* read the second set of memory latencies[4:7] */
2885 val = 1; /* data0 to be programmed to 1 for second set */
2886 ret = sandybridge_pcode_read(dev_priv,
2887 GEN9_PCODE_READ_MEM_LATENCY,
2890 drm_err(&dev_priv->drm,
2891 "SKL Mailbox read error = %d\n", ret);
2895 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2896 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2897 GEN9_MEM_LATENCY_LEVEL_MASK;
2898 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2899 GEN9_MEM_LATENCY_LEVEL_MASK;
2900 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2901 GEN9_MEM_LATENCY_LEVEL_MASK;
2904 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2905 * need to be disabled. We make sure to sanitize the values out
2906 * of the punit to satisfy this requirement.
2908 for (level = 1; level <= max_level; level++) {
2909 if (wm[level] == 0) {
2910 for (i = level + 1; i <= max_level; i++)
2917 * WaWmMemoryReadLatency:skl+,glk
2919 * punit doesn't take into account the read latency so we need
2920 * to add 2us to the various latency levels we retrieve from the
2921 * punit when level 0 response data us 0us.
2925 for (level = 1; level <= max_level; level++) {
2933 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2934 * If we could not get dimm info enable this WA to prevent from
2935 * any underrun. If not able to get Dimm info assume 16GB dimm
2936 * to avoid any underrun.
2938 if (dev_priv->dram_info.is_16gb_dimm)
2941 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2942 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2944 wm[0] = (sskpd >> 56) & 0xFF;
2946 wm[0] = sskpd & 0xF;
2947 wm[1] = (sskpd >> 4) & 0xFF;
2948 wm[2] = (sskpd >> 12) & 0xFF;
2949 wm[3] = (sskpd >> 20) & 0x1FF;
2950 wm[4] = (sskpd >> 32) & 0x1FF;
2951 } else if (INTEL_GEN(dev_priv) >= 6) {
2952 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2954 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2955 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2956 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2957 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2958 } else if (INTEL_GEN(dev_priv) >= 5) {
2959 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2961 /* ILK primary LP0 latency is 700 ns */
2963 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2964 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2966 MISSING_CASE(INTEL_DEVID(dev_priv));
2970 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2973 /* ILK sprite LP0 latency is 1300 ns */
2974 if (IS_GEN(dev_priv, 5))
2978 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2981 /* ILK cursor LP0 latency is 1300 ns */
2982 if (IS_GEN(dev_priv, 5))
2986 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2988 /* how many WM levels are we expecting */
2989 if (INTEL_GEN(dev_priv) >= 9)
2991 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2993 else if (INTEL_GEN(dev_priv) >= 6)
2999 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3003 int level, max_level = ilk_wm_max_level(dev_priv);
3005 for (level = 0; level <= max_level; level++) {
3006 unsigned int latency = wm[level];
3009 drm_dbg_kms(&dev_priv->drm,
3010 "%s WM%d latency not provided\n",
3016 * - latencies are in us on gen9.
3017 * - before then, WM1+ latency values are in 0.5us units
3019 if (INTEL_GEN(dev_priv) >= 9)
3024 drm_dbg_kms(&dev_priv->drm,
3025 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3026 wm[level], latency / 10, latency % 10);
3030 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3033 int level, max_level = ilk_wm_max_level(dev_priv);
3038 wm[0] = max(wm[0], min);
3039 for (level = 1; level <= max_level; level++)
3040 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3045 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3050 * The BIOS provided WM memory latency values are often
3051 * inadequate for high resolution displays. Adjust them.
3053 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3054 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3055 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3060 drm_dbg_kms(&dev_priv->drm,
3061 "WM latency values increased to avoid potential underruns\n");
3062 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3063 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3064 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3067 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3070 * On some SNB machines (Thinkpad X220 Tablet at least)
3071 * LP3 usage can cause vblank interrupts to be lost.
3072 * The DEIIR bit will go high but it looks like the CPU
3073 * never gets interrupted.
3075 * It's not clear whether other interrupt source could
3076 * be affected or if this is somehow limited to vblank
3077 * interrupts only. To play it safe we disable LP3
3078 * watermarks entirely.
3080 if (dev_priv->wm.pri_latency[3] == 0 &&
3081 dev_priv->wm.spr_latency[3] == 0 &&
3082 dev_priv->wm.cur_latency[3] == 0)
3085 dev_priv->wm.pri_latency[3] = 0;
3086 dev_priv->wm.spr_latency[3] = 0;
3087 dev_priv->wm.cur_latency[3] = 0;
3089 drm_dbg_kms(&dev_priv->drm,
3090 "LP3 watermarks disabled due to potential for lost interrupts\n");
3091 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3092 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3093 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3096 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3098 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3100 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3101 sizeof(dev_priv->wm.pri_latency));
3102 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3103 sizeof(dev_priv->wm.pri_latency));
3105 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3106 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3108 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3109 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3110 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3112 if (IS_GEN(dev_priv, 6)) {
3113 snb_wm_latency_quirk(dev_priv);
3114 snb_wm_lp3_irq_quirk(dev_priv);
3118 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3120 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3121 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3124 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3125 struct intel_pipe_wm *pipe_wm)
3127 /* LP0 watermark maximums depend on this pipe alone */
3128 const struct intel_wm_config config = {
3129 .num_pipes_active = 1,
3130 .sprites_enabled = pipe_wm->sprites_enabled,
3131 .sprites_scaled = pipe_wm->sprites_scaled,
3133 struct ilk_wm_maximums max;
3135 /* LP0 watermarks always use 1/2 DDB partitioning */
3136 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3138 /* At least LP0 must be valid */
3139 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3140 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3147 /* Compute new watermarks for the pipe */
3148 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3150 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3151 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3152 struct intel_pipe_wm *pipe_wm;
3153 struct intel_plane *plane;
3154 const struct intel_plane_state *plane_state;
3155 const struct intel_plane_state *pristate = NULL;
3156 const struct intel_plane_state *sprstate = NULL;
3157 const struct intel_plane_state *curstate = NULL;
3158 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3159 struct ilk_wm_maximums max;
3161 pipe_wm = &crtc_state->wm.ilk.optimal;
3163 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3164 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3165 pristate = plane_state;
3166 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3167 sprstate = plane_state;
3168 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3169 curstate = plane_state;
3172 pipe_wm->pipe_enabled = crtc_state->hw.active;
3174 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3175 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3176 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3177 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3180 usable_level = max_level;
3182 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3183 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3186 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3187 if (pipe_wm->sprites_scaled)
3190 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3191 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3192 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3194 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3197 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3199 for (level = 1; level <= usable_level; level++) {
3200 struct intel_wm_level *wm = &pipe_wm->wm[level];
3202 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3203 pristate, sprstate, curstate, wm);
3206 * Disable any watermark level that exceeds the
3207 * register maximums since such watermarks are
3210 if (!ilk_validate_wm_level(level, &max, wm)) {
3211 memset(wm, 0, sizeof(*wm));
3220 * Build a set of 'intermediate' watermark values that satisfy both the old
3221 * state and the new state. These can be programmed to the hardware
3224 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3226 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3227 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3228 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3229 struct intel_atomic_state *intel_state =
3230 to_intel_atomic_state(newstate->uapi.state);
3231 const struct intel_crtc_state *oldstate =
3232 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3233 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3234 int level, max_level = ilk_wm_max_level(dev_priv);
3237 * Start with the final, target watermarks, then combine with the
3238 * currently active watermarks to get values that are safe both before
3239 * and after the vblank.
3241 *a = newstate->wm.ilk.optimal;
3242 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3243 intel_state->skip_intermediate_wm)
3246 a->pipe_enabled |= b->pipe_enabled;
3247 a->sprites_enabled |= b->sprites_enabled;
3248 a->sprites_scaled |= b->sprites_scaled;
3250 for (level = 0; level <= max_level; level++) {
3251 struct intel_wm_level *a_wm = &a->wm[level];
3252 const struct intel_wm_level *b_wm = &b->wm[level];
3254 a_wm->enable &= b_wm->enable;
3255 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3256 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3257 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3258 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3262 * We need to make sure that these merged watermark values are
3263 * actually a valid configuration themselves. If they're not,
3264 * there's no safe way to transition from the old state to
3265 * the new state, so we need to fail the atomic transaction.
3267 if (!ilk_validate_pipe_wm(dev_priv, a))
3271 * If our intermediate WM are identical to the final WM, then we can
3272 * omit the post-vblank programming; only update if it's different.
3274 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3275 newstate->wm.need_postvbl_update = true;
3281 * Merge the watermarks from all active pipes for a specific level.
3283 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3285 struct intel_wm_level *ret_wm)
3287 const struct intel_crtc *intel_crtc;
3289 ret_wm->enable = true;
3291 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3292 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3293 const struct intel_wm_level *wm = &active->wm[level];
3295 if (!active->pipe_enabled)
3299 * The watermark values may have been used in the past,
3300 * so we must maintain them in the registers for some
3301 * time even if the level is now disabled.
3304 ret_wm->enable = false;
3306 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3307 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3308 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3309 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3314 * Merge all low power watermarks for all active pipes.
3316 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3317 const struct intel_wm_config *config,
3318 const struct ilk_wm_maximums *max,
3319 struct intel_pipe_wm *merged)
3321 int level, max_level = ilk_wm_max_level(dev_priv);
3322 int last_enabled_level = max_level;
3324 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3325 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3326 config->num_pipes_active > 1)
3327 last_enabled_level = 0;
3329 /* ILK: FBC WM must be disabled always */
3330 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3332 /* merge each WM1+ level */
3333 for (level = 1; level <= max_level; level++) {
3334 struct intel_wm_level *wm = &merged->wm[level];
3336 ilk_merge_wm_level(dev_priv, level, wm);
3338 if (level > last_enabled_level)
3340 else if (!ilk_validate_wm_level(level, max, wm))
3341 /* make sure all following levels get disabled */
3342 last_enabled_level = level - 1;
3345 * The spec says it is preferred to disable
3346 * FBC WMs instead of disabling a WM level.
3348 if (wm->fbc_val > max->fbc) {
3350 merged->fbc_wm_enabled = false;
3355 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3357 * FIXME this is racy. FBC might get enabled later.
3358 * What we should check here is whether FBC can be
3359 * enabled sometime later.
3361 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3362 intel_fbc_is_active(dev_priv)) {
3363 for (level = 2; level <= max_level; level++) {
3364 struct intel_wm_level *wm = &merged->wm[level];
3371 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3373 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3374 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3377 /* The value we need to program into the WM_LPx latency field */
3378 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3381 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3384 return dev_priv->wm.pri_latency[level];
3387 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3388 const struct intel_pipe_wm *merged,
3389 enum intel_ddb_partitioning partitioning,
3390 struct ilk_wm_values *results)
3392 struct intel_crtc *intel_crtc;
3395 results->enable_fbc_wm = merged->fbc_wm_enabled;
3396 results->partitioning = partitioning;
3398 /* LP1+ register values */
3399 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3400 const struct intel_wm_level *r;
3402 level = ilk_wm_lp_to_level(wm_lp, merged);
3404 r = &merged->wm[level];
3407 * Maintain the watermark values even if the level is
3408 * disabled. Doing otherwise could cause underruns.
3410 results->wm_lp[wm_lp - 1] =
3411 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3412 (r->pri_val << WM1_LP_SR_SHIFT) |
3416 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3418 if (INTEL_GEN(dev_priv) >= 8)
3419 results->wm_lp[wm_lp - 1] |=
3420 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3422 results->wm_lp[wm_lp - 1] |=
3423 r->fbc_val << WM1_LP_FBC_SHIFT;
3426 * Always set WM1S_LP_EN when spr_val != 0, even if the
3427 * level is disabled. Doing otherwise could cause underruns.
3429 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3430 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3431 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3433 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3436 /* LP0 register values */
3437 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3438 enum pipe pipe = intel_crtc->pipe;
3439 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3440 const struct intel_wm_level *r = &pipe_wm->wm[0];
3442 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3445 results->wm_pipe[pipe] =
3446 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3447 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3452 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3453 * case both are at the same level. Prefer r1 in case they're the same. */
3454 static struct intel_pipe_wm *
3455 ilk_find_best_result(struct drm_i915_private *dev_priv,
3456 struct intel_pipe_wm *r1,
3457 struct intel_pipe_wm *r2)
3459 int level, max_level = ilk_wm_max_level(dev_priv);
3460 int level1 = 0, level2 = 0;
3462 for (level = 1; level <= max_level; level++) {
3463 if (r1->wm[level].enable)
3465 if (r2->wm[level].enable)
3469 if (level1 == level2) {
3470 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3474 } else if (level1 > level2) {
3481 /* dirty bits used to track which watermarks need changes */
3482 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3483 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3484 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3485 #define WM_DIRTY_FBC (1 << 24)
3486 #define WM_DIRTY_DDB (1 << 25)
3488 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3489 const struct ilk_wm_values *old,
3490 const struct ilk_wm_values *new)
3492 unsigned int dirty = 0;
3496 for_each_pipe(dev_priv, pipe) {
3497 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3498 dirty |= WM_DIRTY_PIPE(pipe);
3499 /* Must disable LP1+ watermarks too */
3500 dirty |= WM_DIRTY_LP_ALL;
3504 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3505 dirty |= WM_DIRTY_FBC;
3506 /* Must disable LP1+ watermarks too */
3507 dirty |= WM_DIRTY_LP_ALL;
3510 if (old->partitioning != new->partitioning) {
3511 dirty |= WM_DIRTY_DDB;
3512 /* Must disable LP1+ watermarks too */
3513 dirty |= WM_DIRTY_LP_ALL;
3516 /* LP1+ watermarks already deemed dirty, no need to continue */
3517 if (dirty & WM_DIRTY_LP_ALL)
3520 /* Find the lowest numbered LP1+ watermark in need of an update... */
3521 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3522 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3523 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3527 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3528 for (; wm_lp <= 3; wm_lp++)
3529 dirty |= WM_DIRTY_LP(wm_lp);
3534 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3537 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3538 bool changed = false;
3540 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3541 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3542 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3545 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3546 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3547 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3550 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3551 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3552 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3557 * Don't touch WM1S_LP_EN here.
3558 * Doing so could cause underruns.
3565 * The spec says we shouldn't write when we don't need, because every write
3566 * causes WMs to be re-evaluated, expending some power.
3568 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3569 struct ilk_wm_values *results)
3571 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3575 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3579 _ilk_disable_lp_wm(dev_priv, dirty);
3581 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3582 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3583 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3584 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3585 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3586 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3588 if (dirty & WM_DIRTY_DDB) {
3589 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3590 val = I915_READ(WM_MISC);
3591 if (results->partitioning == INTEL_DDB_PART_1_2)
3592 val &= ~WM_MISC_DATA_PARTITION_5_6;
3594 val |= WM_MISC_DATA_PARTITION_5_6;
3595 I915_WRITE(WM_MISC, val);
3597 val = I915_READ(DISP_ARB_CTL2);
3598 if (results->partitioning == INTEL_DDB_PART_1_2)
3599 val &= ~DISP_DATA_PARTITION_5_6;
3601 val |= DISP_DATA_PARTITION_5_6;
3602 I915_WRITE(DISP_ARB_CTL2, val);
3606 if (dirty & WM_DIRTY_FBC) {
3607 val = I915_READ(DISP_ARB_CTL);
3608 if (results->enable_fbc_wm)
3609 val &= ~DISP_FBC_WM_DIS;
3611 val |= DISP_FBC_WM_DIS;
3612 I915_WRITE(DISP_ARB_CTL, val);
3615 if (dirty & WM_DIRTY_LP(1) &&
3616 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3617 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3619 if (INTEL_GEN(dev_priv) >= 7) {
3620 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3621 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3622 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3623 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3626 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3627 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3628 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3629 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3630 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3631 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3633 dev_priv->wm.hw = *results;
3636 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3638 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3641 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3644 int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3645 u8 enabled_slices_mask = 0;
3647 for (i = 0; i < max_slices; i++) {
3648 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3649 enabled_slices_mask |= BIT(i);
3652 return enabled_slices_mask;
3656 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3657 * so assume we'll always need it in order to avoid underruns.
3659 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3661 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3665 intel_has_sagv(struct drm_i915_private *dev_priv)
3667 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3668 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3672 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3674 if (INTEL_GEN(dev_priv) >= 12) {
3678 ret = sandybridge_pcode_read(dev_priv,
3679 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3682 dev_priv->sagv_block_time_us = val;
3686 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3687 } else if (IS_GEN(dev_priv, 11)) {
3688 dev_priv->sagv_block_time_us = 10;
3690 } else if (IS_GEN(dev_priv, 10)) {
3691 dev_priv->sagv_block_time_us = 20;
3693 } else if (IS_GEN(dev_priv, 9)) {
3694 dev_priv->sagv_block_time_us = 30;
3697 MISSING_CASE(INTEL_GEN(dev_priv));
3700 /* Default to an unusable block time */
3701 dev_priv->sagv_block_time_us = -1;
3705 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3706 * depending on power and performance requirements. The display engine access
3707 * to system memory is blocked during the adjustment time. Because of the
3708 * blocking time, having this enabled can cause full system hangs and/or pipe
3709 * underruns if we don't meet all of the following requirements:
3711 * - <= 1 pipe enabled
3712 * - All planes can enable watermarks for latencies >= SAGV engine block time
3713 * - We're not using an interlaced display configuration
3716 intel_enable_sagv(struct drm_i915_private *dev_priv)
3720 if (!intel_has_sagv(dev_priv))
3723 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3726 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3727 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3730 /* We don't need to wait for SAGV when enabling */
3733 * Some skl systems, pre-release machines in particular,
3734 * don't actually have SAGV.
3736 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3737 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3738 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3740 } else if (ret < 0) {
3741 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3745 dev_priv->sagv_status = I915_SAGV_ENABLED;
3750 intel_disable_sagv(struct drm_i915_private *dev_priv)
3754 if (!intel_has_sagv(dev_priv))
3757 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3760 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3761 /* bspec says to keep retrying for at least 1 ms */
3762 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3764 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3767 * Some skl systems, pre-release machines in particular,
3768 * don't actually have SAGV.
3770 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3771 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3772 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3774 } else if (ret < 0) {
3775 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3779 dev_priv->sagv_status = I915_SAGV_DISABLED;
3783 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3785 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3786 const struct intel_bw_state *new_bw_state;
3787 const struct intel_bw_state *old_bw_state;
3791 * Just return if we can't control SAGV or don't have it.
3792 * This is different from situation when we have SAGV but just can't
3793 * afford it due to DBuf limitation - in case if SAGV is completely
3794 * disabled in a BIOS, we are not even allowed to send a PCode request,
3795 * as it will throw an error. So have to check it here.
3797 if (!intel_has_sagv(dev_priv))
3800 new_bw_state = intel_atomic_get_new_bw_state(state);
3804 if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3805 intel_disable_sagv(dev_priv);
3809 old_bw_state = intel_atomic_get_old_bw_state(state);
3813 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3816 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3819 * If new mask is zero - means there is nothing to mask,
3820 * we can only unmask, which should be done in unmask.
3826 * Restrict required qgv points before updating the configuration.
3827 * According to BSpec we can't mask and unmask qgv points at the same
3828 * time. Also masking should be done before updating the configuration
3829 * and unmasking afterwards.
3831 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3834 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3836 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3837 const struct intel_bw_state *new_bw_state;
3838 const struct intel_bw_state *old_bw_state;
3842 * Just return if we can't control SAGV or don't have it.
3843 * This is different from situation when we have SAGV but just can't
3844 * afford it due to DBuf limitation - in case if SAGV is completely
3845 * disabled in a BIOS, we are not even allowed to send a PCode request,
3846 * as it will throw an error. So have to check it here.
3848 if (!intel_has_sagv(dev_priv))
3851 new_bw_state = intel_atomic_get_new_bw_state(state);
3855 if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3856 intel_enable_sagv(dev_priv);
3860 old_bw_state = intel_atomic_get_old_bw_state(state);
3864 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3867 new_mask = new_bw_state->qgv_points_mask;
3870 * Allow required qgv points after updating the configuration.
3871 * According to BSpec we can't mask and unmask qgv points at the same
3872 * time. Also masking should be done before updating the configuration
3873 * and unmasking afterwards.
3875 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3878 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3880 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3881 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3882 struct intel_plane *plane;
3883 const struct intel_plane_state *plane_state;
3886 if (!intel_has_sagv(dev_priv))
3889 if (!crtc_state->hw.active)
3892 if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3895 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3896 const struct skl_plane_wm *wm =
3897 &crtc_state->wm.skl.optimal.planes[plane->id];
3899 /* Skip this plane if it's not enabled */
3900 if (!wm->wm[0].plane_en)
3903 /* Find the highest enabled wm level for this plane */
3904 for (level = ilk_wm_max_level(dev_priv);
3905 !wm->wm[level].plane_en; --level)
3908 latency = dev_priv->wm.skl_latency[level];
3910 if (skl_needs_memory_bw_wa(dev_priv) &&
3911 plane_state->uapi.fb->modifier ==
3912 I915_FORMAT_MOD_X_TILED)
3916 * If any of the planes on this pipe don't enable wm levels that
3917 * incur memory latencies higher than sagv_block_time_us we
3918 * can't enable SAGV.
3920 if (latency < dev_priv->sagv_block_time_us)
3927 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3929 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3930 enum plane_id plane_id;
3932 if (!crtc_state->hw.active)
3935 for_each_plane_id_on_crtc(crtc, plane_id) {
3936 const struct skl_ddb_entry *plane_alloc =
3937 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3938 const struct skl_plane_wm *wm =
3939 &crtc_state->wm.skl.optimal.planes[plane_id];
3941 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3948 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3950 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3951 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3953 if (INTEL_GEN(dev_priv) >= 12)
3954 return tgl_crtc_can_enable_sagv(crtc_state);
3956 return skl_crtc_can_enable_sagv(crtc_state);
3959 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3960 const struct intel_bw_state *bw_state)
3962 if (INTEL_GEN(dev_priv) < 11 &&
3963 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3966 return bw_state->pipe_sagv_reject == 0;
3969 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3971 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3973 struct intel_crtc *crtc;
3974 struct intel_crtc_state *new_crtc_state;
3975 struct intel_bw_state *new_bw_state = NULL;
3976 const struct intel_bw_state *old_bw_state = NULL;
3979 for_each_new_intel_crtc_in_state(state, crtc,
3980 new_crtc_state, i) {
3981 new_bw_state = intel_atomic_get_bw_state(state);
3982 if (IS_ERR(new_bw_state))
3983 return PTR_ERR(new_bw_state);
3985 old_bw_state = intel_atomic_get_old_bw_state(state);
3987 if (intel_crtc_can_enable_sagv(new_crtc_state))
3988 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3990 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3996 new_bw_state->active_pipes =
3997 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3999 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4000 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4005 for_each_new_intel_crtc_in_state(state, crtc,
4006 new_crtc_state, i) {
4007 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4010 * We store use_sagv_wm in the crtc state rather than relying on
4011 * that bw state since we have no convenient way to get at the
4012 * latter from the plane commit hooks (especially in the legacy
4015 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4016 intel_can_enable_sagv(dev_priv, new_bw_state);
4019 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4020 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4021 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4024 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4025 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4034 * Calculate initial DBuf slice offset, based on slice size
4035 * and mask(i.e if slice size is 1024 and second slice is enabled
4036 * offset would be 1024)
4039 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4043 unsigned int offset = 0;
4045 if (!dbuf_slice_mask)
4048 offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4050 WARN_ON(offset >= ddb_size);
4054 u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4056 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4057 drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4059 if (INTEL_GEN(dev_priv) < 11)
4060 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4065 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4066 const struct skl_ddb_entry *entry)
4069 u16 ddb_size = intel_get_ddb_size(dev_priv);
4070 u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4071 u16 slice_size = ddb_size / num_supported_slices;
4075 if (!skl_ddb_entry_size(entry))
4078 start_slice = entry->start / slice_size;
4079 end_slice = (entry->end - 1) / slice_size;
4082 * Per plane DDB entry can in a really worst case be on multiple slices
4083 * but single entry is anyway contigious.
4085 while (start_slice <= end_slice) {
4086 slice_mask |= BIT(start_slice);
4093 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4097 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4098 const struct intel_crtc_state *crtc_state,
4099 const u64 total_data_rate,
4100 struct skl_ddb_entry *alloc, /* out */
4101 int *num_active /* out */)
4103 struct drm_atomic_state *state = crtc_state->uapi.state;
4104 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4105 struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4106 const struct intel_crtc *crtc;
4107 u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4108 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4109 struct intel_dbuf_state *new_dbuf_state =
4110 intel_atomic_get_new_dbuf_state(intel_state);
4111 const struct intel_dbuf_state *old_dbuf_state =
4112 intel_atomic_get_old_dbuf_state(intel_state);
4113 u8 active_pipes = new_dbuf_state->active_pipes;
4117 u32 dbuf_slice_mask;
4120 u32 total_slice_mask;
4124 *num_active = hweight8(active_pipes);
4126 if (!crtc_state->hw.active) {
4132 ddb_size = intel_get_ddb_size(dev_priv);
4134 slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4137 * If the state doesn't change the active CRTC's or there is no
4138 * modeset request, then there's no need to recalculate;
4139 * the existing pipe allocation limits should remain unchanged.
4140 * Note that we're safe from racing commits since any racing commit
4141 * that changes the active CRTC list or do modeset would need to
4142 * grab _all_ crtc locks, including the one we currently hold.
4144 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes &&
4145 !dev_priv->wm.distrust_bios_wm) {
4147 * alloc may be cleared by clear_intel_crtc_state,
4148 * copy from old state to be sure
4150 * FIXME get rid of this mess
4152 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4157 * Get allowed DBuf slices for correspondent pipe and platform.
4159 dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4162 * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4163 * and slice size is 1024, the offset would be 1024
4165 offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4166 slice_size, ddb_size);
4169 * Figure out total size of allowed DBuf slices, which is basically
4170 * a number of allowed slices for that pipe multiplied by slice size.
4172 * range ddb entries are still allocated in proportion to display width.
4174 ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4177 * Watermark/ddb requirement highly depends upon width of the
4178 * framebuffer, So instead of allocating DDB equally among pipes
4179 * distribute DDB based on resolution/width of the display.
4181 total_slice_mask = dbuf_slice_mask;
4182 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4183 const struct drm_display_mode *adjusted_mode =
4184 &crtc_state->hw.adjusted_mode;
4185 enum pipe pipe = crtc->pipe;
4186 int hdisplay, vdisplay;
4187 u32 pipe_dbuf_slice_mask;
4189 if (!crtc_state->hw.active)
4192 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4196 * According to BSpec pipe can share one dbuf slice with another
4197 * pipes or pipe can use multiple dbufs, in both cases we
4198 * account for other pipes only if they have exactly same mask.
4199 * However we need to account how many slices we should enable
4202 total_slice_mask |= pipe_dbuf_slice_mask;
4205 * Do not account pipes using other slice sets
4206 * luckily as of current BSpec slice sets do not partially
4207 * intersect(pipes share either same one slice or same slice set
4208 * i.e no partial intersection), so it is enough to check for
4211 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4214 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4216 total_width_in_range += hdisplay;
4218 if (pipe < for_pipe)
4219 width_before_pipe_in_range += hdisplay;
4220 else if (pipe == for_pipe)
4221 pipe_width = hdisplay;
4225 * FIXME: For now we always enable slice S1 as per
4226 * the Bspec display initialization sequence.
4228 new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1);
4230 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4231 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4236 start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4237 end = ddb_range_size *
4238 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4240 alloc->start = offset + start;
4241 alloc->end = offset + end;
4243 drm_dbg_kms(&dev_priv->drm,
4244 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
4245 for_crtc->base.id, for_crtc->name,
4246 dbuf_slice_mask, alloc->start, alloc->end, active_pipes);
4251 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4252 int width, const struct drm_format_info *format,
4253 u64 modifier, unsigned int rotation,
4254 u32 plane_pixel_rate, struct skl_wm_params *wp,
4256 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4258 unsigned int latency,
4259 const struct skl_wm_params *wp,
4260 const struct skl_wm_level *result_prev,
4261 struct skl_wm_level *result /* out */);
4264 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4267 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4268 int level, max_level = ilk_wm_max_level(dev_priv);
4269 struct skl_wm_level wm = {};
4270 int ret, min_ddb_alloc = 0;
4271 struct skl_wm_params wp;
4273 ret = skl_compute_wm_params(crtc_state, 256,
4274 drm_format_info(DRM_FORMAT_ARGB8888),
4275 DRM_FORMAT_MOD_LINEAR,
4277 crtc_state->pixel_rate, &wp, 0);
4278 drm_WARN_ON(&dev_priv->drm, ret);
4280 for (level = 0; level <= max_level; level++) {
4281 unsigned int latency = dev_priv->wm.skl_latency[level];
4283 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4284 if (wm.min_ddb_alloc == U16_MAX)
4287 min_ddb_alloc = wm.min_ddb_alloc;
4290 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4293 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4294 struct skl_ddb_entry *entry, u32 reg)
4297 entry->start = reg & DDB_ENTRY_MASK;
4298 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4305 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4306 const enum pipe pipe,
4307 const enum plane_id plane_id,
4308 struct skl_ddb_entry *ddb_y,
4309 struct skl_ddb_entry *ddb_uv)
4314 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4315 if (plane_id == PLANE_CURSOR) {
4316 val = I915_READ(CUR_BUF_CFG(pipe));
4317 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4321 val = I915_READ(PLANE_CTL(pipe, plane_id));
4323 /* No DDB allocated for disabled planes */
4324 if (val & PLANE_CTL_ENABLE)
4325 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4326 val & PLANE_CTL_ORDER_RGBX,
4327 val & PLANE_CTL_ALPHA_MASK);
4329 if (INTEL_GEN(dev_priv) >= 11) {
4330 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4331 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4333 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4334 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4337 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4340 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4341 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4345 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4346 struct skl_ddb_entry *ddb_y,
4347 struct skl_ddb_entry *ddb_uv)
4349 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4350 enum intel_display_power_domain power_domain;
4351 enum pipe pipe = crtc->pipe;
4352 intel_wakeref_t wakeref;
4353 enum plane_id plane_id;
4355 power_domain = POWER_DOMAIN_PIPE(pipe);
4356 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4360 for_each_plane_id_on_crtc(crtc, plane_id)
4361 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4366 intel_display_power_put(dev_priv, power_domain, wakeref);
4370 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4371 * The bspec defines downscale amount as:
4374 * Horizontal down scale amount = maximum[1, Horizontal source size /
4375 * Horizontal destination size]
4376 * Vertical down scale amount = maximum[1, Vertical source size /
4377 * Vertical destination size]
4378 * Total down scale amount = Horizontal down scale amount *
4379 * Vertical down scale amount
4382 * Return value is provided in 16.16 fixed point form to retain fractional part.
4383 * Caller should take care of dividing & rounding off the value.
4385 static uint_fixed_16_16_t
4386 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4387 const struct intel_plane_state *plane_state)
4389 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4390 u32 src_w, src_h, dst_w, dst_h;
4391 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4392 uint_fixed_16_16_t downscale_h, downscale_w;
4394 if (drm_WARN_ON(&dev_priv->drm,
4395 !intel_wm_plane_visible(crtc_state, plane_state)))
4396 return u32_to_fixed16(0);
4399 * Src coordinates are already rotated by 270 degrees for
4400 * the 90/270 degree plane rotation cases (to match the
4401 * GTT mapping), hence no need to account for rotation here.
4403 * n.b., src is 16.16 fixed point, dst is whole integer.
4405 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4406 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4407 dst_w = drm_rect_width(&plane_state->uapi.dst);
4408 dst_h = drm_rect_height(&plane_state->uapi.dst);
4410 fp_w_ratio = div_fixed16(src_w, dst_w);
4411 fp_h_ratio = div_fixed16(src_h, dst_h);
4412 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4413 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4415 return mul_fixed16(downscale_w, downscale_h);
4418 struct dbuf_slice_conf_entry {
4420 u8 dbuf_mask[I915_MAX_PIPES];
4424 * Table taken from Bspec 12716
4425 * Pipes do have some preferred DBuf slice affinity,
4426 * plus there are some hardcoded requirements on how
4427 * those should be distributed for multipipe scenarios.
4428 * For more DBuf slices algorithm can get even more messy
4429 * and less readable, so decided to use a table almost
4430 * as is from BSpec itself - that way it is at least easier
4431 * to compare, change and check.
4433 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4434 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4437 .active_pipes = BIT(PIPE_A),
4439 [PIPE_A] = BIT(DBUF_S1),
4443 .active_pipes = BIT(PIPE_B),
4445 [PIPE_B] = BIT(DBUF_S1),
4449 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4451 [PIPE_A] = BIT(DBUF_S1),
4452 [PIPE_B] = BIT(DBUF_S2),
4456 .active_pipes = BIT(PIPE_C),
4458 [PIPE_C] = BIT(DBUF_S2),
4462 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4464 [PIPE_A] = BIT(DBUF_S1),
4465 [PIPE_C] = BIT(DBUF_S2),
4469 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4471 [PIPE_B] = BIT(DBUF_S1),
4472 [PIPE_C] = BIT(DBUF_S2),
4476 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4478 [PIPE_A] = BIT(DBUF_S1),
4479 [PIPE_B] = BIT(DBUF_S1),
4480 [PIPE_C] = BIT(DBUF_S2),
4487 * Table taken from Bspec 49255
4488 * Pipes do have some preferred DBuf slice affinity,
4489 * plus there are some hardcoded requirements on how
4490 * those should be distributed for multipipe scenarios.
4491 * For more DBuf slices algorithm can get even more messy
4492 * and less readable, so decided to use a table almost
4493 * as is from BSpec itself - that way it is at least easier
4494 * to compare, change and check.
4496 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4497 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4500 .active_pipes = BIT(PIPE_A),
4502 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4506 .active_pipes = BIT(PIPE_B),
4508 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4512 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4514 [PIPE_A] = BIT(DBUF_S2),
4515 [PIPE_B] = BIT(DBUF_S1),
4519 .active_pipes = BIT(PIPE_C),
4521 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4525 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4527 [PIPE_A] = BIT(DBUF_S1),
4528 [PIPE_C] = BIT(DBUF_S2),
4532 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4534 [PIPE_B] = BIT(DBUF_S1),
4535 [PIPE_C] = BIT(DBUF_S2),
4539 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4541 [PIPE_A] = BIT(DBUF_S1),
4542 [PIPE_B] = BIT(DBUF_S1),
4543 [PIPE_C] = BIT(DBUF_S2),
4547 .active_pipes = BIT(PIPE_D),
4549 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4553 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4555 [PIPE_A] = BIT(DBUF_S1),
4556 [PIPE_D] = BIT(DBUF_S2),
4560 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4562 [PIPE_B] = BIT(DBUF_S1),
4563 [PIPE_D] = BIT(DBUF_S2),
4567 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4569 [PIPE_A] = BIT(DBUF_S1),
4570 [PIPE_B] = BIT(DBUF_S1),
4571 [PIPE_D] = BIT(DBUF_S2),
4575 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4577 [PIPE_C] = BIT(DBUF_S1),
4578 [PIPE_D] = BIT(DBUF_S2),
4582 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4584 [PIPE_A] = BIT(DBUF_S1),
4585 [PIPE_C] = BIT(DBUF_S2),
4586 [PIPE_D] = BIT(DBUF_S2),
4590 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4592 [PIPE_B] = BIT(DBUF_S1),
4593 [PIPE_C] = BIT(DBUF_S2),
4594 [PIPE_D] = BIT(DBUF_S2),
4598 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4600 [PIPE_A] = BIT(DBUF_S1),
4601 [PIPE_B] = BIT(DBUF_S1),
4602 [PIPE_C] = BIT(DBUF_S2),
4603 [PIPE_D] = BIT(DBUF_S2),
4609 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4610 const struct dbuf_slice_conf_entry *dbuf_slices)
4614 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4615 if (dbuf_slices[i].active_pipes == active_pipes)
4616 return dbuf_slices[i].dbuf_mask[pipe];
4622 * This function finds an entry with same enabled pipe configuration and
4623 * returns correspondent DBuf slice mask as stated in BSpec for particular
4626 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4629 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4630 * required calculating "pipe ratio" in order to determine
4631 * if one or two slices can be used for single pipe configurations
4632 * as additional constraint to the existing table.
4633 * However based on recent info, it should be not "pipe ratio"
4634 * but rather ratio between pixel_rate and cdclk with additional
4635 * constants, so for now we are using only table until this is
4636 * clarified. Also this is the reason why crtc_state param is
4637 * still here - we will need it once those additional constraints
4640 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4643 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4645 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4648 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4651 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4652 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4653 enum pipe pipe = crtc->pipe;
4655 if (IS_GEN(dev_priv, 12))
4656 return tgl_compute_dbuf_slices(pipe, active_pipes);
4657 else if (IS_GEN(dev_priv, 11))
4658 return icl_compute_dbuf_slices(pipe, active_pipes);
4660 * For anything else just return one slice yet.
4661 * Should be extended for other platforms.
4663 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4667 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4668 const struct intel_plane_state *plane_state,
4671 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4672 const struct drm_framebuffer *fb = plane_state->hw.fb;
4674 u32 width = 0, height = 0;
4675 uint_fixed_16_16_t down_scale_amount;
4678 if (!plane_state->uapi.visible)
4681 if (plane->id == PLANE_CURSOR)
4684 if (color_plane == 1 &&
4685 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4689 * Src coordinates are already rotated by 270 degrees for
4690 * the 90/270 degree plane rotation cases (to match the
4691 * GTT mapping), hence no need to account for rotation here.
4693 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4694 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4696 /* UV plane does 1/2 pixel sub-sampling */
4697 if (color_plane == 1) {
4702 data_rate = width * height;
4704 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4706 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4708 rate *= fb->format->cpp[color_plane];
4713 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4714 u64 *plane_data_rate,
4715 u64 *uv_plane_data_rate)
4717 struct intel_plane *plane;
4718 const struct intel_plane_state *plane_state;
4719 u64 total_data_rate = 0;
4721 /* Calculate and cache data rate for each plane */
4722 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4723 enum plane_id plane_id = plane->id;
4727 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4728 plane_data_rate[plane_id] = rate;
4729 total_data_rate += rate;
4732 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4733 uv_plane_data_rate[plane_id] = rate;
4734 total_data_rate += rate;
4737 return total_data_rate;
4741 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4742 u64 *plane_data_rate)
4744 struct intel_plane *plane;
4745 const struct intel_plane_state *plane_state;
4746 u64 total_data_rate = 0;
4748 /* Calculate and cache data rate for each plane */
4749 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4750 enum plane_id plane_id = plane->id;
4753 if (!plane_state->planar_linked_plane) {
4754 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4755 plane_data_rate[plane_id] = rate;
4756 total_data_rate += rate;
4758 enum plane_id y_plane_id;
4761 * The slave plane might not iterate in
4762 * intel_atomic_crtc_state_for_each_plane_state(),
4763 * and needs the master plane state which may be
4764 * NULL if we try get_new_plane_state(), so we
4765 * always calculate from the master.
4767 if (plane_state->planar_slave)
4770 /* Y plane rate is calculated on the slave */
4771 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4772 y_plane_id = plane_state->planar_linked_plane->id;
4773 plane_data_rate[y_plane_id] = rate;
4774 total_data_rate += rate;
4776 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4777 plane_data_rate[plane_id] = rate;
4778 total_data_rate += rate;
4782 return total_data_rate;
4785 static const struct skl_wm_level *
4786 skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4787 enum plane_id plane_id,
4790 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4791 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4793 if (level == 0 && pipe_wm->use_sagv_wm)
4794 return &wm->sagv_wm0;
4796 return &wm->wm[level];
4800 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4802 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4803 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4804 struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4805 u16 alloc_size, start = 0;
4806 u16 total[I915_MAX_PLANES] = {};
4807 u16 uv_total[I915_MAX_PLANES] = {};
4808 u64 total_data_rate;
4809 enum plane_id plane_id;
4811 u64 plane_data_rate[I915_MAX_PLANES] = {};
4812 u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4817 /* Clear the partitioning for disabled planes. */
4818 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4819 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4821 if (!crtc_state->hw.active) {
4822 struct intel_atomic_state *state =
4823 to_intel_atomic_state(crtc_state->uapi.state);
4824 struct intel_dbuf_state *new_dbuf_state =
4825 intel_atomic_get_new_dbuf_state(state);
4826 const struct intel_dbuf_state *old_dbuf_state =
4827 intel_atomic_get_old_dbuf_state(state);
4830 * FIXME hack to make sure we compute this sensibly when
4831 * turning off all the pipes. Otherwise we leave it at
4832 * whatever we had previously, and then runtime PM will
4833 * mess it up by turning off all but S1. Remove this
4834 * once the dbuf state computation flow becomes sane.
4836 if (new_dbuf_state->active_pipes == 0) {
4837 new_dbuf_state->enabled_slices = BIT(DBUF_S1);
4839 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4840 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4846 alloc->start = alloc->end = 0;
4850 if (INTEL_GEN(dev_priv) >= 11)
4852 icl_get_total_relative_data_rate(crtc_state,
4856 skl_get_total_relative_data_rate(crtc_state,
4858 uv_plane_data_rate);
4860 ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state,
4862 alloc, &num_active);
4866 alloc_size = skl_ddb_entry_size(alloc);
4867 if (alloc_size == 0)
4870 /* Allocate fixed number of blocks for cursor. */
4871 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4872 alloc_size -= total[PLANE_CURSOR];
4873 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4874 alloc->end - total[PLANE_CURSOR];
4875 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4877 if (total_data_rate == 0)
4881 * Find the highest watermark level for which we can satisfy the block
4882 * requirement of active planes.
4884 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4886 for_each_plane_id_on_crtc(crtc, plane_id) {
4887 const struct skl_plane_wm *wm =
4888 &crtc_state->wm.skl.optimal.planes[plane_id];
4890 if (plane_id == PLANE_CURSOR) {
4891 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4892 drm_WARN_ON(&dev_priv->drm,
4893 wm->wm[level].min_ddb_alloc != U16_MAX);
4900 blocks += wm->wm[level].min_ddb_alloc;
4901 blocks += wm->uv_wm[level].min_ddb_alloc;
4904 if (blocks <= alloc_size) {
4905 alloc_size -= blocks;
4911 drm_dbg_kms(&dev_priv->drm,
4912 "Requested display configuration exceeds system DDB limitations");
4913 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4914 blocks, alloc_size);
4919 * Grant each plane the blocks it requires at the highest achievable
4920 * watermark level, plus an extra share of the leftover blocks
4921 * proportional to its relative data rate.
4923 for_each_plane_id_on_crtc(crtc, plane_id) {
4924 const struct skl_plane_wm *wm =
4925 &crtc_state->wm.skl.optimal.planes[plane_id];
4929 if (plane_id == PLANE_CURSOR)
4933 * We've accounted for all active planes; remaining planes are
4936 if (total_data_rate == 0)
4939 rate = plane_data_rate[plane_id];
4940 extra = min_t(u16, alloc_size,
4941 DIV64_U64_ROUND_UP(alloc_size * rate,
4943 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4944 alloc_size -= extra;
4945 total_data_rate -= rate;
4947 if (total_data_rate == 0)
4950 rate = uv_plane_data_rate[plane_id];
4951 extra = min_t(u16, alloc_size,
4952 DIV64_U64_ROUND_UP(alloc_size * rate,
4954 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4955 alloc_size -= extra;
4956 total_data_rate -= rate;
4958 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4960 /* Set the actual DDB start/end points for each plane */
4961 start = alloc->start;
4962 for_each_plane_id_on_crtc(crtc, plane_id) {
4963 struct skl_ddb_entry *plane_alloc =
4964 &crtc_state->wm.skl.plane_ddb_y[plane_id];
4965 struct skl_ddb_entry *uv_plane_alloc =
4966 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4968 if (plane_id == PLANE_CURSOR)
4971 /* Gen11+ uses a separate plane for UV watermarks */
4972 drm_WARN_ON(&dev_priv->drm,
4973 INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4975 /* Leave disabled planes at (0,0) */
4976 if (total[plane_id]) {
4977 plane_alloc->start = start;
4978 start += total[plane_id];
4979 plane_alloc->end = start;
4982 if (uv_total[plane_id]) {
4983 uv_plane_alloc->start = start;
4984 start += uv_total[plane_id];
4985 uv_plane_alloc->end = start;
4990 * When we calculated watermark values we didn't know how high
4991 * of a level we'd actually be able to hit, so we just marked
4992 * all levels as "enabled." Go back now and disable the ones
4993 * that aren't actually possible.
4995 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4996 for_each_plane_id_on_crtc(crtc, plane_id) {
4997 struct skl_plane_wm *wm =
4998 &crtc_state->wm.skl.optimal.planes[plane_id];
5001 * We only disable the watermarks for each plane if
5002 * they exceed the ddb allocation of said plane. This
5003 * is done so that we don't end up touching cursor
5004 * watermarks needlessly when some other plane reduces
5005 * our max possible watermark level.
5007 * Bspec has this to say about the PLANE_WM enable bit:
5008 * "All the watermarks at this level for all enabled
5009 * planes must be enabled before the level will be used."
5010 * So this is actually safe to do.
5012 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
5013 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
5014 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
5017 * Wa_1408961008:icl, ehl
5018 * Underruns with WM1+ disabled
5020 if (IS_GEN(dev_priv, 11) &&
5021 level == 1 && wm->wm[0].plane_en) {
5022 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
5023 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
5024 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5030 * Go back and disable the transition watermark if it turns out we
5031 * don't have enough DDB blocks for it.
5033 for_each_plane_id_on_crtc(crtc, plane_id) {
5034 struct skl_plane_wm *wm =
5035 &crtc_state->wm.skl.optimal.planes[plane_id];
5037 if (wm->trans_wm.plane_res_b >= total[plane_id])
5038 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
5045 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5046 * for the read latency) and cpp should always be <= 8, so that
5047 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5048 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5050 static uint_fixed_16_16_t
5051 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5052 u8 cpp, u32 latency, u32 dbuf_block_size)
5054 u32 wm_intermediate_val;
5055 uint_fixed_16_16_t ret;
5058 return FP_16_16_MAX;
5060 wm_intermediate_val = latency * pixel_rate * cpp;
5061 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5063 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5064 ret = add_fixed16_u32(ret, 1);
5069 static uint_fixed_16_16_t
5070 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5071 uint_fixed_16_16_t plane_blocks_per_line)
5073 u32 wm_intermediate_val;
5074 uint_fixed_16_16_t ret;
5077 return FP_16_16_MAX;
5079 wm_intermediate_val = latency * pixel_rate;
5080 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5081 pipe_htotal * 1000);
5082 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5086 static uint_fixed_16_16_t
5087 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5089 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5092 uint_fixed_16_16_t linetime_us;
5094 if (!crtc_state->hw.active)
5095 return u32_to_fixed16(0);
5097 pixel_rate = crtc_state->pixel_rate;
5099 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5100 return u32_to_fixed16(0);
5102 crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5103 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5109 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5110 const struct intel_plane_state *plane_state)
5112 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5113 u64 adjusted_pixel_rate;
5114 uint_fixed_16_16_t downscale_amount;
5116 /* Shouldn't reach here on disabled planes... */
5117 if (drm_WARN_ON(&dev_priv->drm,
5118 !intel_wm_plane_visible(crtc_state, plane_state)))
5122 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5123 * with additional adjustments for plane-specific scaling.
5125 adjusted_pixel_rate = crtc_state->pixel_rate;
5126 downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5128 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5133 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5134 int width, const struct drm_format_info *format,
5135 u64 modifier, unsigned int rotation,
5136 u32 plane_pixel_rate, struct skl_wm_params *wp,
5139 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5140 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5143 /* only planar format has two planes */
5144 if (color_plane == 1 &&
5145 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5146 drm_dbg_kms(&dev_priv->drm,
5147 "Non planar format have single plane\n");
5151 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5152 modifier == I915_FORMAT_MOD_Yf_TILED ||
5153 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5154 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5155 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5156 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5157 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5158 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5161 if (color_plane == 1 && wp->is_planar)
5164 wp->cpp = format->cpp[color_plane];
5165 wp->plane_pixel_rate = plane_pixel_rate;
5167 if (INTEL_GEN(dev_priv) >= 11 &&
5168 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5169 wp->dbuf_block_size = 256;
5171 wp->dbuf_block_size = 512;
5173 if (drm_rotation_90_or_270(rotation)) {
5176 wp->y_min_scanlines = 16;
5179 wp->y_min_scanlines = 8;
5182 wp->y_min_scanlines = 4;
5185 MISSING_CASE(wp->cpp);
5189 wp->y_min_scanlines = 4;
5192 if (skl_needs_memory_bw_wa(dev_priv))
5193 wp->y_min_scanlines *= 2;
5195 wp->plane_bytes_per_line = wp->width * wp->cpp;
5197 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5198 wp->y_min_scanlines,
5199 wp->dbuf_block_size);
5201 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5204 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5205 wp->y_min_scanlines);
5207 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5208 wp->dbuf_block_size);
5211 INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5214 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5217 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5218 wp->plane_blocks_per_line);
5220 wp->linetime_us = fixed16_to_u32_round_up(
5221 intel_get_linetime_us(crtc_state));
5227 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5228 const struct intel_plane_state *plane_state,
5229 struct skl_wm_params *wp, int color_plane)
5231 const struct drm_framebuffer *fb = plane_state->hw.fb;
5235 * Src coordinates are already rotated by 270 degrees for
5236 * the 90/270 degree plane rotation cases (to match the
5237 * GTT mapping), hence no need to account for rotation here.
5239 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5241 return skl_compute_wm_params(crtc_state, width,
5242 fb->format, fb->modifier,
5243 plane_state->hw.rotation,
5244 skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5248 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5250 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5253 /* The number of lines are ignored for the level 0 watermark. */
5257 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5259 unsigned int latency,
5260 const struct skl_wm_params *wp,
5261 const struct skl_wm_level *result_prev,
5262 struct skl_wm_level *result /* out */)
5264 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5265 uint_fixed_16_16_t method1, method2;
5266 uint_fixed_16_16_t selected_result;
5267 u32 res_blocks, res_lines, min_ddb_alloc = 0;
5271 result->min_ddb_alloc = U16_MAX;
5276 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5277 * Display WA #1141: kbl,cfl
5279 if ((IS_KABYLAKE(dev_priv) ||
5280 IS_COFFEELAKE(dev_priv) ||
5281 IS_COMETLAKE(dev_priv)) &&
5282 dev_priv->ipc_enabled)
5285 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5288 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5289 wp->cpp, latency, wp->dbuf_block_size);
5290 method2 = skl_wm_method2(wp->plane_pixel_rate,
5291 crtc_state->hw.adjusted_mode.crtc_htotal,
5293 wp->plane_blocks_per_line);
5296 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5298 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5299 wp->dbuf_block_size < 1) &&
5300 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5301 selected_result = method2;
5302 } else if (latency >= wp->linetime_us) {
5303 if (IS_GEN(dev_priv, 9) &&
5304 !IS_GEMINILAKE(dev_priv))
5305 selected_result = min_fixed16(method1, method2);
5307 selected_result = method2;
5309 selected_result = method1;
5313 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5314 res_lines = div_round_up_fixed16(selected_result,
5315 wp->plane_blocks_per_line);
5317 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5318 /* Display WA #1125: skl,bxt,kbl */
5319 if (level == 0 && wp->rc_surface)
5321 fixed16_to_u32_round_up(wp->y_tile_minimum);
5323 /* Display WA #1126: skl,bxt,kbl */
5324 if (level >= 1 && level <= 7) {
5327 fixed16_to_u32_round_up(wp->y_tile_minimum);
5328 res_lines += wp->y_min_scanlines;
5334 * Make sure result blocks for higher latency levels are
5335 * atleast as high as level below the current level.
5336 * Assumption in DDB algorithm optimization for special
5337 * cases. Also covers Display WA #1125 for RC.
5339 if (result_prev->plane_res_b > res_blocks)
5340 res_blocks = result_prev->plane_res_b;
5344 if (INTEL_GEN(dev_priv) >= 11) {
5348 if (res_lines % wp->y_min_scanlines == 0)
5349 extra_lines = wp->y_min_scanlines;
5351 extra_lines = wp->y_min_scanlines * 2 -
5352 res_lines % wp->y_min_scanlines;
5354 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5355 wp->plane_blocks_per_line);
5357 min_ddb_alloc = res_blocks +
5358 DIV_ROUND_UP(res_blocks, 10);
5362 if (!skl_wm_has_lines(dev_priv, level))
5365 if (res_lines > 31) {
5367 result->min_ddb_alloc = U16_MAX;
5372 * If res_lines is valid, assume we can use this watermark level
5373 * for now. We'll come back and disable it after we calculate the
5374 * DDB allocation if it turns out we don't actually have enough
5375 * blocks to satisfy it.
5377 result->plane_res_b = res_blocks;
5378 result->plane_res_l = res_lines;
5379 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5380 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5381 result->plane_en = true;
5385 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5386 const struct skl_wm_params *wm_params,
5387 struct skl_wm_level *levels)
5389 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5390 int level, max_level = ilk_wm_max_level(dev_priv);
5391 struct skl_wm_level *result_prev = &levels[0];
5393 for (level = 0; level <= max_level; level++) {
5394 struct skl_wm_level *result = &levels[level];
5395 unsigned int latency = dev_priv->wm.skl_latency[level];
5397 skl_compute_plane_wm(crtc_state, level, latency,
5398 wm_params, result_prev, result);
5400 result_prev = result;
5404 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5405 const struct skl_wm_params *wm_params,
5406 struct skl_plane_wm *plane_wm)
5408 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5409 struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5410 struct skl_wm_level *levels = plane_wm->wm;
5411 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5413 skl_compute_plane_wm(crtc_state, 0, latency,
5414 wm_params, &levels[0],
5418 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5419 const struct skl_wm_params *wp,
5420 struct skl_plane_wm *wm)
5422 struct drm_device *dev = crtc_state->uapi.crtc->dev;
5423 const struct drm_i915_private *dev_priv = to_i915(dev);
5424 u16 trans_min, trans_amount, trans_y_tile_min;
5425 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5427 /* Transition WM don't make any sense if ipc is disabled */
5428 if (!dev_priv->ipc_enabled)
5432 * WaDisableTWM:skl,kbl,cfl,bxt
5433 * Transition WM are not recommended by HW team for GEN9
5435 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5438 if (INTEL_GEN(dev_priv) >= 11)
5443 /* Display WA #1140: glk,cnl */
5444 if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5447 trans_amount = 10; /* This is configurable amount */
5449 trans_offset_b = trans_min + trans_amount;
5452 * The spec asks for Selected Result Blocks for wm0 (the real value),
5453 * not Result Blocks (the integer value). Pay attention to the capital
5454 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5455 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5456 * and since we later will have to get the ceiling of the sum in the
5457 * transition watermarks calculation, we can just pretend Selected
5458 * Result Blocks is Result Blocks minus 1 and it should work for the
5459 * current platforms.
5461 wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5465 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5466 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5469 res_blocks = wm0_sel_res_b + trans_offset_b;
5473 * Just assume we can enable the transition watermark. After
5474 * computing the DDB we'll come back and disable it if that
5475 * assumption turns out to be false.
5477 wm->trans_wm.plane_res_b = res_blocks + 1;
5478 wm->trans_wm.plane_en = true;
5481 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5482 const struct intel_plane_state *plane_state,
5483 enum plane_id plane_id, int color_plane)
5485 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5486 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5487 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5488 struct skl_wm_params wm_params;
5491 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5492 &wm_params, color_plane);
5496 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5498 if (INTEL_GEN(dev_priv) >= 12)
5499 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5501 skl_compute_transition_wm(crtc_state, &wm_params, wm);
5506 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5507 const struct intel_plane_state *plane_state,
5508 enum plane_id plane_id)
5510 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5511 struct skl_wm_params wm_params;
5514 wm->is_planar = true;
5516 /* uv plane watermarks must also be validated for NV12/Planar */
5517 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5522 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5527 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5528 const struct intel_plane_state *plane_state)
5530 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5531 const struct drm_framebuffer *fb = plane_state->hw.fb;
5532 enum plane_id plane_id = plane->id;
5535 if (!intel_wm_plane_visible(crtc_state, plane_state))
5538 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5543 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5544 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5553 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5554 const struct intel_plane_state *plane_state)
5556 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5557 enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5560 /* Watermarks calculated in master */
5561 if (plane_state->planar_slave)
5564 if (plane_state->planar_linked_plane) {
5565 const struct drm_framebuffer *fb = plane_state->hw.fb;
5566 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5568 drm_WARN_ON(&dev_priv->drm,
5569 !intel_wm_plane_visible(crtc_state, plane_state));
5570 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5571 fb->format->num_planes == 1);
5573 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5578 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5582 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5583 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5592 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5594 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5595 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5596 struct intel_plane *plane;
5597 const struct intel_plane_state *plane_state;
5601 * We'll only calculate watermarks for planes that are actually
5602 * enabled, so make sure all other planes are set as disabled.
5604 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5606 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5609 if (INTEL_GEN(dev_priv) >= 11)
5610 ret = icl_build_plane_wm(crtc_state, plane_state);
5612 ret = skl_build_plane_wm(crtc_state, plane_state);
5620 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5622 const struct skl_ddb_entry *entry)
5625 intel_de_write_fw(dev_priv, reg,
5626 (entry->end - 1) << 16 | entry->start);
5628 intel_de_write_fw(dev_priv, reg, 0);
5631 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5633 const struct skl_wm_level *level)
5637 if (level->plane_en)
5639 if (level->ignore_lines)
5640 val |= PLANE_WM_IGNORE_LINES;
5641 val |= level->plane_res_b;
5642 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5644 intel_de_write_fw(dev_priv, reg, val);
5647 void skl_write_plane_wm(struct intel_plane *plane,
5648 const struct intel_crtc_state *crtc_state)
5650 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5651 int level, max_level = ilk_wm_max_level(dev_priv);
5652 enum plane_id plane_id = plane->id;
5653 enum pipe pipe = plane->pipe;
5654 const struct skl_plane_wm *wm =
5655 &crtc_state->wm.skl.optimal.planes[plane_id];
5656 const struct skl_ddb_entry *ddb_y =
5657 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5658 const struct skl_ddb_entry *ddb_uv =
5659 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5661 for (level = 0; level <= max_level; level++) {
5662 const struct skl_wm_level *wm_level;
5664 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5666 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5669 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5672 if (INTEL_GEN(dev_priv) >= 11) {
5673 skl_ddb_entry_write(dev_priv,
5674 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5679 swap(ddb_y, ddb_uv);
5681 skl_ddb_entry_write(dev_priv,
5682 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5683 skl_ddb_entry_write(dev_priv,
5684 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5687 void skl_write_cursor_wm(struct intel_plane *plane,
5688 const struct intel_crtc_state *crtc_state)
5690 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5691 int level, max_level = ilk_wm_max_level(dev_priv);
5692 enum plane_id plane_id = plane->id;
5693 enum pipe pipe = plane->pipe;
5694 const struct skl_plane_wm *wm =
5695 &crtc_state->wm.skl.optimal.planes[plane_id];
5696 const struct skl_ddb_entry *ddb =
5697 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5699 for (level = 0; level <= max_level; level++) {
5700 const struct skl_wm_level *wm_level;
5702 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5704 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5707 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5709 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5712 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5713 const struct skl_wm_level *l2)
5715 return l1->plane_en == l2->plane_en &&
5716 l1->ignore_lines == l2->ignore_lines &&
5717 l1->plane_res_l == l2->plane_res_l &&
5718 l1->plane_res_b == l2->plane_res_b;
5721 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5722 const struct skl_plane_wm *wm1,
5723 const struct skl_plane_wm *wm2)
5725 int level, max_level = ilk_wm_max_level(dev_priv);
5727 for (level = 0; level <= max_level; level++) {
5729 * We don't check uv_wm as the hardware doesn't actually
5730 * use it. It only gets used for calculating the required
5733 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5737 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5740 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5741 const struct skl_ddb_entry *b)
5743 return a->start < b->end && b->start < a->end;
5746 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5747 const struct skl_ddb_entry *entries,
5748 int num_entries, int ignore_idx)
5752 for (i = 0; i < num_entries; i++) {
5753 if (i != ignore_idx &&
5754 skl_ddb_entries_overlap(ddb, &entries[i]))
5762 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5763 struct intel_crtc_state *new_crtc_state)
5765 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5766 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5767 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5768 struct intel_plane *plane;
5770 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5771 struct intel_plane_state *plane_state;
5772 enum plane_id plane_id = plane->id;
5774 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5775 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5776 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5777 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5780 plane_state = intel_atomic_get_plane_state(state, plane);
5781 if (IS_ERR(plane_state))
5782 return PTR_ERR(plane_state);
5784 new_crtc_state->update_planes |= BIT(plane_id);
5791 skl_compute_ddb(struct intel_atomic_state *state)
5793 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5794 const struct intel_dbuf_state *old_dbuf_state;
5795 const struct intel_dbuf_state *new_dbuf_state;
5796 const struct intel_crtc_state *old_crtc_state;
5797 struct intel_crtc_state *new_crtc_state;
5798 struct intel_crtc *crtc;
5801 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5802 new_crtc_state, i) {
5803 ret = skl_allocate_pipe_ddb(new_crtc_state);
5807 ret = skl_ddb_add_affected_planes(old_crtc_state,
5813 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5814 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
5816 if (new_dbuf_state &&
5817 new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices)
5818 drm_dbg_kms(&dev_priv->drm,
5819 "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5820 old_dbuf_state->enabled_slices,
5821 new_dbuf_state->enabled_slices,
5822 INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5827 static char enast(bool enable)
5829 return enable ? '*' : ' ';
5833 skl_print_wm_changes(struct intel_atomic_state *state)
5835 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5836 const struct intel_crtc_state *old_crtc_state;
5837 const struct intel_crtc_state *new_crtc_state;
5838 struct intel_plane *plane;
5839 struct intel_crtc *crtc;
5842 if (!drm_debug_enabled(DRM_UT_KMS))
5845 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5846 new_crtc_state, i) {
5847 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5849 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5850 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5852 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5853 enum plane_id plane_id = plane->id;
5854 const struct skl_ddb_entry *old, *new;
5856 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5857 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5859 if (skl_ddb_entry_equal(old, new))
5862 drm_dbg_kms(&dev_priv->drm,
5863 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5864 plane->base.base.id, plane->base.name,
5865 old->start, old->end, new->start, new->end,
5866 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5869 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5870 enum plane_id plane_id = plane->id;
5871 const struct skl_plane_wm *old_wm, *new_wm;
5873 old_wm = &old_pipe_wm->planes[plane_id];
5874 new_wm = &new_pipe_wm->planes[plane_id];
5876 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5879 drm_dbg_kms(&dev_priv->drm,
5880 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5881 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5882 plane->base.base.id, plane->base.name,
5883 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5884 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5885 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5886 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5887 enast(old_wm->trans_wm.plane_en),
5888 enast(old_wm->sagv_wm0.plane_en),
5889 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5890 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5891 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5892 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5893 enast(new_wm->trans_wm.plane_en),
5894 enast(new_wm->sagv_wm0.plane_en));
5896 drm_dbg_kms(&dev_priv->drm,
5897 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5898 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5899 plane->base.base.id, plane->base.name,
5900 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5901 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5902 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5903 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5904 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5905 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5906 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5907 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5908 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5909 enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5911 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5912 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5913 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5914 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5915 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5916 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5917 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5918 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5919 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5920 enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5922 drm_dbg_kms(&dev_priv->drm,
5923 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5924 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5925 plane->base.base.id, plane->base.name,
5926 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5927 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5928 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5929 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5930 old_wm->trans_wm.plane_res_b,
5931 old_wm->sagv_wm0.plane_res_b,
5932 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5933 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5934 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5935 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5936 new_wm->trans_wm.plane_res_b,
5937 new_wm->sagv_wm0.plane_res_b);
5939 drm_dbg_kms(&dev_priv->drm,
5940 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5941 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5942 plane->base.base.id, plane->base.name,
5943 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5944 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5945 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5946 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5947 old_wm->trans_wm.min_ddb_alloc,
5948 old_wm->sagv_wm0.min_ddb_alloc,
5949 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5950 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5951 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5952 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5953 new_wm->trans_wm.min_ddb_alloc,
5954 new_wm->sagv_wm0.min_ddb_alloc);
5959 static int intel_add_affected_pipes(struct intel_atomic_state *state,
5962 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5963 struct intel_crtc *crtc;
5965 for_each_intel_crtc(&dev_priv->drm, crtc) {
5966 struct intel_crtc_state *crtc_state;
5968 if ((pipe_mask & BIT(crtc->pipe)) == 0)
5971 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5972 if (IS_ERR(crtc_state))
5973 return PTR_ERR(crtc_state);
5980 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5982 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5983 struct intel_crtc_state *crtc_state;
5984 struct intel_crtc *crtc;
5987 if (dev_priv->wm.distrust_bios_wm) {
5989 * skl_ddb_get_pipe_allocation_limits() currently requires
5990 * all active pipes to be included in the state so that
5991 * it can redistribute the dbuf among them, and it really
5992 * wants to recompute things when distrust_bios_wm is set
5993 * so we add all the pipes to the state.
5995 ret = intel_add_affected_pipes(state, ~0);
6000 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6001 struct intel_dbuf_state *new_dbuf_state;
6002 const struct intel_dbuf_state *old_dbuf_state;
6004 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6005 if (IS_ERR(new_dbuf_state))
6006 return PTR_ERR(new_dbuf_state);
6008 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6010 new_dbuf_state->active_pipes =
6011 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6013 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes)
6016 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6021 * skl_ddb_get_pipe_allocation_limits() currently requires
6022 * all active pipes to be included in the state so that
6023 * it can redistribute the dbuf among them.
6025 ret = intel_add_affected_pipes(state,
6026 new_dbuf_state->active_pipes);
6037 * To make sure the cursor watermark registers are always consistent
6038 * with our computed state the following scenario needs special
6042 * 2. move cursor entirely offscreen
6045 * Step 2. does call .disable_plane() but does not zero the watermarks
6046 * (since we consider an offscreen cursor still active for the purposes
6047 * of watermarks). Step 3. would not normally call .disable_plane()
6048 * because the actual plane visibility isn't changing, and we don't
6049 * deallocate the cursor ddb until the pipe gets disabled. So we must
6050 * force step 3. to call .disable_plane() to update the watermark
6051 * registers properly.
6053 * Other planes do not suffer from this issues as their watermarks are
6054 * calculated based on the actual plane visibility. The only time this
6055 * can trigger for the other planes is during the initial readout as the
6056 * default value of the watermarks registers is not zero.
6058 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6059 struct intel_crtc *crtc)
6061 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6062 const struct intel_crtc_state *old_crtc_state =
6063 intel_atomic_get_old_crtc_state(state, crtc);
6064 struct intel_crtc_state *new_crtc_state =
6065 intel_atomic_get_new_crtc_state(state, crtc);
6066 struct intel_plane *plane;
6068 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6069 struct intel_plane_state *plane_state;
6070 enum plane_id plane_id = plane->id;
6073 * Force a full wm update for every plane on modeset.
6074 * Required because the reset value of the wm registers
6075 * is non-zero, whereas we want all disabled planes to
6076 * have zero watermarks. So if we turn off the relevant
6077 * power well the hardware state will go out of sync
6078 * with the software state.
6080 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6081 skl_plane_wm_equals(dev_priv,
6082 &old_crtc_state->wm.skl.optimal.planes[plane_id],
6083 &new_crtc_state->wm.skl.optimal.planes[plane_id]))
6086 plane_state = intel_atomic_get_plane_state(state, plane);
6087 if (IS_ERR(plane_state))
6088 return PTR_ERR(plane_state);
6090 new_crtc_state->update_planes |= BIT(plane_id);
6097 skl_compute_wm(struct intel_atomic_state *state)
6099 struct intel_crtc *crtc;
6100 struct intel_crtc_state *new_crtc_state;
6101 struct intel_crtc_state *old_crtc_state;
6104 ret = skl_ddb_add_affected_pipes(state);
6109 * Calculate WM's for all pipes that are part of this transaction.
6110 * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6111 * weren't otherwise being modified if pipe allocations had to change.
6113 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6114 new_crtc_state, i) {
6115 ret = skl_build_pipe_wm(new_crtc_state);
6120 ret = skl_compute_ddb(state);
6124 ret = intel_compute_sagv_mask(state);
6129 * skl_compute_ddb() will have adjusted the final watermarks
6130 * based on how much ddb is available. Now we can actually
6131 * check if the final watermarks changed.
6133 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6134 new_crtc_state, i) {
6135 ret = skl_wm_add_affected_planes(state, crtc);
6140 skl_print_wm_changes(state);
6145 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6146 struct intel_wm_config *config)
6148 struct intel_crtc *crtc;
6150 /* Compute the currently _active_ config */
6151 for_each_intel_crtc(&dev_priv->drm, crtc) {
6152 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6154 if (!wm->pipe_enabled)
6157 config->sprites_enabled |= wm->sprites_enabled;
6158 config->sprites_scaled |= wm->sprites_scaled;
6159 config->num_pipes_active++;
6163 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6165 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6166 struct ilk_wm_maximums max;
6167 struct intel_wm_config config = {};
6168 struct ilk_wm_values results = {};
6169 enum intel_ddb_partitioning partitioning;
6171 ilk_compute_wm_config(dev_priv, &config);
6173 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6174 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6176 /* 5/6 split only in single pipe config on IVB+ */
6177 if (INTEL_GEN(dev_priv) >= 7 &&
6178 config.num_pipes_active == 1 && config.sprites_enabled) {
6179 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6180 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6182 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6184 best_lp_wm = &lp_wm_1_2;
6187 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6188 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6190 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6192 ilk_write_wm_values(dev_priv, &results);
6195 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6196 struct intel_crtc *crtc)
6198 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6199 const struct intel_crtc_state *crtc_state =
6200 intel_atomic_get_new_crtc_state(state, crtc);
6202 mutex_lock(&dev_priv->wm.wm_mutex);
6203 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6204 ilk_program_watermarks(dev_priv);
6205 mutex_unlock(&dev_priv->wm.wm_mutex);
6208 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6209 struct intel_crtc *crtc)
6211 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6212 const struct intel_crtc_state *crtc_state =
6213 intel_atomic_get_new_crtc_state(state, crtc);
6215 if (!crtc_state->wm.need_postvbl_update)
6218 mutex_lock(&dev_priv->wm.wm_mutex);
6219 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6220 ilk_program_watermarks(dev_priv);
6221 mutex_unlock(&dev_priv->wm.wm_mutex);
6224 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6226 level->plane_en = val & PLANE_WM_EN;
6227 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6228 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6229 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6230 PLANE_WM_LINES_MASK;
6233 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6234 struct skl_pipe_wm *out)
6236 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6237 enum pipe pipe = crtc->pipe;
6238 int level, max_level;
6239 enum plane_id plane_id;
6242 max_level = ilk_wm_max_level(dev_priv);
6244 for_each_plane_id_on_crtc(crtc, plane_id) {
6245 struct skl_plane_wm *wm = &out->planes[plane_id];
6247 for (level = 0; level <= max_level; level++) {
6248 if (plane_id != PLANE_CURSOR)
6249 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6251 val = I915_READ(CUR_WM(pipe, level));
6253 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6256 if (INTEL_GEN(dev_priv) >= 12)
6257 wm->sagv_wm0 = wm->wm[0];
6259 if (plane_id != PLANE_CURSOR)
6260 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6262 val = I915_READ(CUR_WM_TRANS(pipe));
6264 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6271 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6273 struct intel_crtc *crtc;
6274 struct intel_crtc_state *crtc_state;
6276 for_each_intel_crtc(&dev_priv->drm, crtc) {
6277 crtc_state = to_intel_crtc_state(crtc->base.state);
6279 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6282 if (dev_priv->active_pipes) {
6283 /* Fully recompute DDB on first atomic commit */
6284 dev_priv->wm.distrust_bios_wm = true;
6288 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6290 struct drm_device *dev = crtc->base.dev;
6291 struct drm_i915_private *dev_priv = to_i915(dev);
6292 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6293 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6294 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6295 enum pipe pipe = crtc->pipe;
6296 static const i915_reg_t wm0_pipe_reg[] = {
6297 [PIPE_A] = WM0_PIPEA_ILK,
6298 [PIPE_B] = WM0_PIPEB_ILK,
6299 [PIPE_C] = WM0_PIPEC_IVB,
6302 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6304 memset(active, 0, sizeof(*active));
6306 active->pipe_enabled = crtc->active;
6308 if (active->pipe_enabled) {
6309 u32 tmp = hw->wm_pipe[pipe];
6312 * For active pipes LP0 watermark is marked as
6313 * enabled, and LP1+ watermaks as disabled since
6314 * we can't really reverse compute them in case
6315 * multiple pipes are active.
6317 active->wm[0].enable = true;
6318 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6319 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6320 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6322 int level, max_level = ilk_wm_max_level(dev_priv);
6325 * For inactive pipes, all watermark levels
6326 * should be marked as enabled but zeroed,
6327 * which is what we'd compute them to.
6329 for (level = 0; level <= max_level; level++)
6330 active->wm[level].enable = true;
6333 crtc->wm.active.ilk = *active;
6336 #define _FW_WM(value, plane) \
6337 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6338 #define _FW_WM_VLV(value, plane) \
6339 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6341 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6342 struct g4x_wm_values *wm)
6346 tmp = I915_READ(DSPFW1);
6347 wm->sr.plane = _FW_WM(tmp, SR);
6348 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6349 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6350 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6352 tmp = I915_READ(DSPFW2);
6353 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6354 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6355 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6356 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6357 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6358 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6360 tmp = I915_READ(DSPFW3);
6361 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6362 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6363 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6364 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6367 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6368 struct vlv_wm_values *wm)
6373 for_each_pipe(dev_priv, pipe) {
6374 tmp = I915_READ(VLV_DDL(pipe));
6376 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6377 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6378 wm->ddl[pipe].plane[PLANE_CURSOR] =
6379 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6380 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6381 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6382 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6383 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6386 tmp = I915_READ(DSPFW1);
6387 wm->sr.plane = _FW_WM(tmp, SR);
6388 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6389 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6390 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6392 tmp = I915_READ(DSPFW2);
6393 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6394 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6395 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6397 tmp = I915_READ(DSPFW3);
6398 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6400 if (IS_CHERRYVIEW(dev_priv)) {
6401 tmp = I915_READ(DSPFW7_CHV);
6402 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6403 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6405 tmp = I915_READ(DSPFW8_CHV);
6406 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6407 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6409 tmp = I915_READ(DSPFW9_CHV);
6410 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6411 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6413 tmp = I915_READ(DSPHOWM);
6414 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6415 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6416 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6417 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6418 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6419 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6420 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6421 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6422 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6423 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6425 tmp = I915_READ(DSPFW7);
6426 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6427 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6429 tmp = I915_READ(DSPHOWM);
6430 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6431 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6432 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6433 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6434 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6435 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6436 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6443 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6445 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6446 struct intel_crtc *crtc;
6448 g4x_read_wm_values(dev_priv, wm);
6450 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6452 for_each_intel_crtc(&dev_priv->drm, crtc) {
6453 struct intel_crtc_state *crtc_state =
6454 to_intel_crtc_state(crtc->base.state);
6455 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6456 struct g4x_pipe_wm *raw;
6457 enum pipe pipe = crtc->pipe;
6458 enum plane_id plane_id;
6459 int level, max_level;
6461 active->cxsr = wm->cxsr;
6462 active->hpll_en = wm->hpll_en;
6463 active->fbc_en = wm->fbc_en;
6465 active->sr = wm->sr;
6466 active->hpll = wm->hpll;
6468 for_each_plane_id_on_crtc(crtc, plane_id) {
6469 active->wm.plane[plane_id] =
6470 wm->pipe[pipe].plane[plane_id];
6473 if (wm->cxsr && wm->hpll_en)
6474 max_level = G4X_WM_LEVEL_HPLL;
6476 max_level = G4X_WM_LEVEL_SR;
6478 max_level = G4X_WM_LEVEL_NORMAL;
6480 level = G4X_WM_LEVEL_NORMAL;
6481 raw = &crtc_state->wm.g4x.raw[level];
6482 for_each_plane_id_on_crtc(crtc, plane_id)
6483 raw->plane[plane_id] = active->wm.plane[plane_id];
6485 if (++level > max_level)
6488 raw = &crtc_state->wm.g4x.raw[level];
6489 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6490 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6491 raw->plane[PLANE_SPRITE0] = 0;
6492 raw->fbc = active->sr.fbc;
6494 if (++level > max_level)
6497 raw = &crtc_state->wm.g4x.raw[level];
6498 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6499 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6500 raw->plane[PLANE_SPRITE0] = 0;
6501 raw->fbc = active->hpll.fbc;
6504 for_each_plane_id_on_crtc(crtc, plane_id)
6505 g4x_raw_plane_wm_set(crtc_state, level,
6506 plane_id, USHRT_MAX);
6507 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6509 crtc_state->wm.g4x.optimal = *active;
6510 crtc_state->wm.g4x.intermediate = *active;
6512 drm_dbg_kms(&dev_priv->drm,
6513 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6515 wm->pipe[pipe].plane[PLANE_PRIMARY],
6516 wm->pipe[pipe].plane[PLANE_CURSOR],
6517 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6520 drm_dbg_kms(&dev_priv->drm,
6521 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6522 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6523 drm_dbg_kms(&dev_priv->drm,
6524 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6525 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6526 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6527 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6530 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6532 struct intel_plane *plane;
6533 struct intel_crtc *crtc;
6535 mutex_lock(&dev_priv->wm.wm_mutex);
6537 for_each_intel_plane(&dev_priv->drm, plane) {
6538 struct intel_crtc *crtc =
6539 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6540 struct intel_crtc_state *crtc_state =
6541 to_intel_crtc_state(crtc->base.state);
6542 struct intel_plane_state *plane_state =
6543 to_intel_plane_state(plane->base.state);
6544 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6545 enum plane_id plane_id = plane->id;
6548 if (plane_state->uapi.visible)
6551 for (level = 0; level < 3; level++) {
6552 struct g4x_pipe_wm *raw =
6553 &crtc_state->wm.g4x.raw[level];
6555 raw->plane[plane_id] = 0;
6556 wm_state->wm.plane[plane_id] = 0;
6559 if (plane_id == PLANE_PRIMARY) {
6560 for (level = 0; level < 3; level++) {
6561 struct g4x_pipe_wm *raw =
6562 &crtc_state->wm.g4x.raw[level];
6566 wm_state->sr.fbc = 0;
6567 wm_state->hpll.fbc = 0;
6568 wm_state->fbc_en = false;
6572 for_each_intel_crtc(&dev_priv->drm, crtc) {
6573 struct intel_crtc_state *crtc_state =
6574 to_intel_crtc_state(crtc->base.state);
6576 crtc_state->wm.g4x.intermediate =
6577 crtc_state->wm.g4x.optimal;
6578 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6581 g4x_program_watermarks(dev_priv);
6583 mutex_unlock(&dev_priv->wm.wm_mutex);
6586 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6588 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6589 struct intel_crtc *crtc;
6592 vlv_read_wm_values(dev_priv, wm);
6594 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6595 wm->level = VLV_WM_LEVEL_PM2;
6597 if (IS_CHERRYVIEW(dev_priv)) {
6598 vlv_punit_get(dev_priv);
6600 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6601 if (val & DSP_MAXFIFO_PM5_ENABLE)
6602 wm->level = VLV_WM_LEVEL_PM5;
6605 * If DDR DVFS is disabled in the BIOS, Punit
6606 * will never ack the request. So if that happens
6607 * assume we don't have to enable/disable DDR DVFS
6608 * dynamically. To test that just set the REQ_ACK
6609 * bit to poke the Punit, but don't change the
6610 * HIGH/LOW bits so that we don't actually change
6611 * the current state.
6613 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6614 val |= FORCE_DDR_FREQ_REQ_ACK;
6615 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6617 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6618 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6619 drm_dbg_kms(&dev_priv->drm,
6620 "Punit not acking DDR DVFS request, "
6621 "assuming DDR DVFS is disabled\n");
6622 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6624 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6625 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6626 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6629 vlv_punit_put(dev_priv);
6632 for_each_intel_crtc(&dev_priv->drm, crtc) {
6633 struct intel_crtc_state *crtc_state =
6634 to_intel_crtc_state(crtc->base.state);
6635 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6636 const struct vlv_fifo_state *fifo_state =
6637 &crtc_state->wm.vlv.fifo_state;
6638 enum pipe pipe = crtc->pipe;
6639 enum plane_id plane_id;
6642 vlv_get_fifo_size(crtc_state);
6644 active->num_levels = wm->level + 1;
6645 active->cxsr = wm->cxsr;
6647 for (level = 0; level < active->num_levels; level++) {
6648 struct g4x_pipe_wm *raw =
6649 &crtc_state->wm.vlv.raw[level];
6651 active->sr[level].plane = wm->sr.plane;
6652 active->sr[level].cursor = wm->sr.cursor;
6654 for_each_plane_id_on_crtc(crtc, plane_id) {
6655 active->wm[level].plane[plane_id] =
6656 wm->pipe[pipe].plane[plane_id];
6658 raw->plane[plane_id] =
6659 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6660 fifo_state->plane[plane_id]);
6664 for_each_plane_id_on_crtc(crtc, plane_id)
6665 vlv_raw_plane_wm_set(crtc_state, level,
6666 plane_id, USHRT_MAX);
6667 vlv_invalidate_wms(crtc, active, level);
6669 crtc_state->wm.vlv.optimal = *active;
6670 crtc_state->wm.vlv.intermediate = *active;
6672 drm_dbg_kms(&dev_priv->drm,
6673 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6675 wm->pipe[pipe].plane[PLANE_PRIMARY],
6676 wm->pipe[pipe].plane[PLANE_CURSOR],
6677 wm->pipe[pipe].plane[PLANE_SPRITE0],
6678 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6681 drm_dbg_kms(&dev_priv->drm,
6682 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6683 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6686 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6688 struct intel_plane *plane;
6689 struct intel_crtc *crtc;
6691 mutex_lock(&dev_priv->wm.wm_mutex);
6693 for_each_intel_plane(&dev_priv->drm, plane) {
6694 struct intel_crtc *crtc =
6695 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6696 struct intel_crtc_state *crtc_state =
6697 to_intel_crtc_state(crtc->base.state);
6698 struct intel_plane_state *plane_state =
6699 to_intel_plane_state(plane->base.state);
6700 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6701 const struct vlv_fifo_state *fifo_state =
6702 &crtc_state->wm.vlv.fifo_state;
6703 enum plane_id plane_id = plane->id;
6706 if (plane_state->uapi.visible)
6709 for (level = 0; level < wm_state->num_levels; level++) {
6710 struct g4x_pipe_wm *raw =
6711 &crtc_state->wm.vlv.raw[level];
6713 raw->plane[plane_id] = 0;
6715 wm_state->wm[level].plane[plane_id] =
6716 vlv_invert_wm_value(raw->plane[plane_id],
6717 fifo_state->plane[plane_id]);
6721 for_each_intel_crtc(&dev_priv->drm, crtc) {
6722 struct intel_crtc_state *crtc_state =
6723 to_intel_crtc_state(crtc->base.state);
6725 crtc_state->wm.vlv.intermediate =
6726 crtc_state->wm.vlv.optimal;
6727 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6730 vlv_program_watermarks(dev_priv);
6732 mutex_unlock(&dev_priv->wm.wm_mutex);
6736 * FIXME should probably kill this and improve
6737 * the real watermark readout/sanitation instead
6739 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6741 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6742 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6743 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6746 * Don't touch WM1S_LP_EN here.
6747 * Doing so could cause underruns.
6751 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6753 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6754 struct intel_crtc *crtc;
6756 ilk_init_lp_watermarks(dev_priv);
6758 for_each_intel_crtc(&dev_priv->drm, crtc)
6759 ilk_pipe_wm_get_hw_state(crtc);
6761 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6762 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6763 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6765 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6766 if (INTEL_GEN(dev_priv) >= 7) {
6767 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6768 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6771 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6772 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6773 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6774 else if (IS_IVYBRIDGE(dev_priv))
6775 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6776 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6779 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6783 * intel_update_watermarks - update FIFO watermark values based on current modes
6784 * @crtc: the #intel_crtc on which to compute the WM
6786 * Calculate watermark values for the various WM regs based on current mode
6787 * and plane configuration.
6789 * There are several cases to deal with here:
6790 * - normal (i.e. non-self-refresh)
6791 * - self-refresh (SR) mode
6792 * - lines are large relative to FIFO size (buffer can hold up to 2)
6793 * - lines are small relative to FIFO size (buffer can hold more than 2
6794 * lines), so need to account for TLB latency
6796 * The normal calculation is:
6797 * watermark = dotclock * bytes per pixel * latency
6798 * where latency is platform & configuration dependent (we assume pessimal
6801 * The SR calculation is:
6802 * watermark = (trunc(latency/line time)+1) * surface width *
6805 * line time = htotal / dotclock
6806 * surface width = hdisplay for normal plane and 64 for cursor
6807 * and latency is assumed to be high, as above.
6809 * The final value programmed to the register should always be rounded up,
6810 * and include an extra 2 entries to account for clock crossings.
6812 * We don't use the sprite, so we can ignore that. And on Crestline we have
6813 * to set the non-SR watermarks to 8.
6815 void intel_update_watermarks(struct intel_crtc *crtc)
6817 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6819 if (dev_priv->display.update_wm)
6820 dev_priv->display.update_wm(crtc);
6823 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6827 if (!HAS_IPC(dev_priv))
6830 val = I915_READ(DISP_ARB_CTL2);
6832 if (dev_priv->ipc_enabled)
6833 val |= DISP_IPC_ENABLE;
6835 val &= ~DISP_IPC_ENABLE;
6837 I915_WRITE(DISP_ARB_CTL2, val);
6840 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6842 /* Display WA #0477 WaDisableIPC: skl */
6843 if (IS_SKYLAKE(dev_priv))
6846 /* Display WA #1141: SKL:all KBL:all CFL */
6847 if (IS_KABYLAKE(dev_priv) ||
6848 IS_COFFEELAKE(dev_priv) ||
6849 IS_COMETLAKE(dev_priv))
6850 return dev_priv->dram_info.symmetric_memory;
6855 void intel_init_ipc(struct drm_i915_private *dev_priv)
6857 if (!HAS_IPC(dev_priv))
6860 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6862 intel_enable_ipc(dev_priv);
6865 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6868 * On Ibex Peak and Cougar Point, we need to disable clock
6869 * gating for the panel power sequencer or it will fail to
6870 * start up when no ports are active.
6872 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6875 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6879 for_each_pipe(dev_priv, pipe) {
6880 I915_WRITE(DSPCNTR(pipe),
6881 I915_READ(DSPCNTR(pipe)) |
6882 DISPPLANE_TRICKLE_FEED_DISABLE);
6884 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6885 POSTING_READ(DSPSURF(pipe));
6889 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6891 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6895 * WaFbcDisableDpfcClockGating:ilk
6897 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6898 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6899 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6901 I915_WRITE(PCH_3DCGDIS0,
6902 MARIUNIT_CLOCK_GATE_DISABLE |
6903 SVSMUNIT_CLOCK_GATE_DISABLE);
6904 I915_WRITE(PCH_3DCGDIS1,
6905 VFMUNIT_CLOCK_GATE_DISABLE);
6908 * According to the spec the following bits should be set in
6909 * order to enable memory self-refresh
6910 * The bit 22/21 of 0x42004
6911 * The bit 5 of 0x42020
6912 * The bit 15 of 0x45000
6914 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6915 (I915_READ(ILK_DISPLAY_CHICKEN2) |
6916 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6917 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6918 I915_WRITE(DISP_ARB_CTL,
6919 (I915_READ(DISP_ARB_CTL) |
6923 * Based on the document from hardware guys the following bits
6924 * should be set unconditionally in order to enable FBC.
6925 * The bit 22 of 0x42000
6926 * The bit 22 of 0x42004
6927 * The bit 7,8,9 of 0x42020.
6929 if (IS_IRONLAKE_M(dev_priv)) {
6930 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6931 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6932 I915_READ(ILK_DISPLAY_CHICKEN1) |
6934 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6935 I915_READ(ILK_DISPLAY_CHICKEN2) |
6939 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6941 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6942 I915_READ(ILK_DISPLAY_CHICKEN2) |
6943 ILK_ELPIN_409_SELECT);
6945 g4x_disable_trickle_feed(dev_priv);
6947 ibx_init_clock_gating(dev_priv);
6950 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6956 * On Ibex Peak and Cougar Point, we need to disable clock
6957 * gating for the panel power sequencer or it will fail to
6958 * start up when no ports are active.
6960 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6961 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6962 PCH_CPUNIT_CLOCK_GATE_DISABLE);
6963 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6964 DPLS_EDP_PPS_FIX_DIS);
6965 /* The below fixes the weird display corruption, a few pixels shifted
6966 * downward, on (only) LVDS of some HP laptops with IVY.
6968 for_each_pipe(dev_priv, pipe) {
6969 val = I915_READ(TRANS_CHICKEN2(pipe));
6970 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6971 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6972 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6973 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6974 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6975 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6976 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6978 /* WADP0ClockGatingDisable */
6979 for_each_pipe(dev_priv, pipe) {
6980 I915_WRITE(TRANS_CHICKEN1(pipe),
6981 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6985 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6989 tmp = I915_READ(MCH_SSKPD);
6990 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6991 drm_dbg_kms(&dev_priv->drm,
6992 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6996 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6998 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7000 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
7002 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7003 I915_READ(ILK_DISPLAY_CHICKEN2) |
7004 ILK_ELPIN_409_SELECT);
7006 I915_WRITE(GEN6_UCGCTL1,
7007 I915_READ(GEN6_UCGCTL1) |
7008 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7009 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7011 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7012 * gating disable must be set. Failure to set it results in
7013 * flickering pixels due to Z write ordering failures after
7014 * some amount of runtime in the Mesa "fire" demo, and Unigine
7015 * Sanctuary and Tropics, and apparently anything else with
7016 * alpha test or pixel discard.
7018 * According to the spec, bit 11 (RCCUNIT) must also be set,
7019 * but we didn't debug actual testcases to find it out.
7021 * WaDisableRCCUnitClockGating:snb
7022 * WaDisableRCPBUnitClockGating:snb
7024 I915_WRITE(GEN6_UCGCTL2,
7025 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7026 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7029 * According to the spec the following bits should be
7030 * set in order to enable memory self-refresh and fbc:
7031 * The bit21 and bit22 of 0x42000
7032 * The bit21 and bit22 of 0x42004
7033 * The bit5 and bit7 of 0x42020
7034 * The bit14 of 0x70180
7035 * The bit14 of 0x71180
7037 * WaFbcAsynchFlipDisableFbcQueue:snb
7039 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7040 I915_READ(ILK_DISPLAY_CHICKEN1) |
7041 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7042 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7043 I915_READ(ILK_DISPLAY_CHICKEN2) |
7044 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7045 I915_WRITE(ILK_DSPCLK_GATE_D,
7046 I915_READ(ILK_DSPCLK_GATE_D) |
7047 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7048 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7050 g4x_disable_trickle_feed(dev_priv);
7052 cpt_init_clock_gating(dev_priv);
7054 gen6_check_mch_setup(dev_priv);
7057 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7060 * TODO: this bit should only be enabled when really needed, then
7061 * disabled when not needed anymore in order to save power.
7063 if (HAS_PCH_LPT_LP(dev_priv))
7064 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7065 I915_READ(SOUTH_DSPCLK_GATE_D) |
7066 PCH_LP_PARTITION_LEVEL_DISABLE);
7068 /* WADPOClockGatingDisable:hsw */
7069 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7070 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7071 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7074 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7076 if (HAS_PCH_LPT_LP(dev_priv)) {
7077 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7079 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7080 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7084 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7085 int general_prio_credits,
7086 int high_prio_credits)
7091 /* WaTempDisableDOPClkGating:bdw */
7092 misccpctl = I915_READ(GEN7_MISCCPCTL);
7093 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7095 val = I915_READ(GEN8_L3SQCREG1);
7096 val &= ~L3_PRIO_CREDITS_MASK;
7097 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7098 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7099 I915_WRITE(GEN8_L3SQCREG1, val);
7102 * Wait at least 100 clocks before re-enabling clock gating.
7103 * See the definition of L3SQCREG1 in BSpec.
7105 POSTING_READ(GEN8_L3SQCREG1);
7107 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7110 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7112 /* Wa_1409120013:icl,ehl */
7113 I915_WRITE(ILK_DPFC_CHICKEN,
7114 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7116 /* This is not an Wa. Enable to reduce Sampler power */
7117 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7118 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7120 /*Wa_14010594013:icl, ehl */
7121 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7122 0, CNL_DELAY_PMRSP);
7125 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7127 u32 vd_pg_enable = 0;
7130 /* Wa_1409120013:tgl */
7131 I915_WRITE(ILK_DPFC_CHICKEN,
7132 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7134 /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7135 for (i = 0; i < I915_MAX_VCS; i++) {
7136 if (HAS_ENGINE(&dev_priv->gt, _VCS(i)))
7137 vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7138 VDN_MFX_POWERGATE_ENABLE(i);
7141 I915_WRITE(POWERGATE_ENABLE,
7142 I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7144 /* Wa_1409825376:tgl (pre-prod)*/
7145 if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
7146 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7147 TGL_VRH_GATING_DIS);
7149 /* Wa_14011059788:tgl */
7150 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7154 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7156 if (!HAS_PCH_CNP(dev_priv))
7159 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7160 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7161 CNP_PWM_CGE_GATING_DISABLE);
7164 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7167 cnp_init_clock_gating(dev_priv);
7169 /* This is not an Wa. Enable for better image quality */
7170 I915_WRITE(_3D_CHICKEN3,
7171 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7173 /* WaEnableChickenDCPR:cnl */
7174 I915_WRITE(GEN8_CHICKEN_DCPR_1,
7175 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7178 * WaFbcWakeMemOn:cnl
7179 * Display WA #0859: cnl
7181 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7182 DISP_FBC_MEMORY_WAKE);
7184 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7185 /* ReadHitWriteOnlyDisable:cnl */
7186 val |= RCCUNIT_CLKGATE_DIS;
7187 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7189 /* Wa_2201832410:cnl */
7190 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7191 val |= GWUNIT_CLKGATE_DIS;
7192 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7194 /* WaDisableVFclkgate:cnl */
7195 /* WaVFUnitClockGatingDisable:cnl */
7196 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7197 val |= VFUNIT_CLKGATE_DIS;
7198 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7201 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7203 cnp_init_clock_gating(dev_priv);
7204 gen9_init_clock_gating(dev_priv);
7207 * WaFbcTurnOffFbcWatermark:cfl
7208 * Display WA #0562: cfl
7210 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7214 * WaFbcNukeOnHostModify:cfl
7215 * Display WA #0873: cfl
7217 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7218 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7221 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7223 gen9_init_clock_gating(dev_priv);
7225 /* WaDisableSDEUnitClockGating:kbl */
7226 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7227 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7228 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7230 /* WaDisableGamClockGating:kbl */
7231 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7232 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7233 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7236 * WaFbcTurnOffFbcWatermark:kbl
7237 * Display WA #0562: kbl
7239 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7243 * WaFbcNukeOnHostModify:kbl
7244 * Display WA #0873: kbl
7246 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7247 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7250 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7252 gen9_init_clock_gating(dev_priv);
7254 /* WAC6entrylatency:skl */
7255 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7256 FBC_LLC_FULLY_OPEN);
7259 * WaFbcTurnOffFbcWatermark:skl
7260 * Display WA #0562: skl
7262 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7266 * WaFbcNukeOnHostModify:skl
7267 * Display WA #0873: skl
7269 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7270 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7273 * WaFbcHighMemBwCorruptionAvoidance:skl
7274 * Display WA #0883: skl
7276 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7277 ILK_DPFC_DISABLE_DUMMY0);
7280 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7284 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7285 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7286 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7289 /* WaSwitchSolVfFArbitrationPriority:bdw */
7290 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7292 /* WaPsrDPAMaskVBlankInSRD:bdw */
7293 I915_WRITE(CHICKEN_PAR1_1,
7294 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7296 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7297 for_each_pipe(dev_priv, pipe) {
7298 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7299 I915_READ(CHICKEN_PIPESL_1(pipe)) |
7300 BDW_DPRS_MASK_VBLANK_SRD);
7303 /* WaVSRefCountFullforceMissDisable:bdw */
7304 /* WaDSRefCountFullforceMissDisable:bdw */
7305 I915_WRITE(GEN7_FF_THREAD_MODE,
7306 I915_READ(GEN7_FF_THREAD_MODE) &
7307 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7309 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7310 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7312 /* WaDisableSDEUnitClockGating:bdw */
7313 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7314 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7316 /* WaProgramL3SqcReg1Default:bdw */
7317 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7319 /* WaKVMNotificationOnConfigChange:bdw */
7320 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7321 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7323 lpt_init_clock_gating(dev_priv);
7325 /* WaDisableDopClockGating:bdw
7327 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7330 I915_WRITE(GEN6_UCGCTL1,
7331 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7334 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7336 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7337 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7338 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7341 /* This is required by WaCatErrorRejectionIssue:hsw */
7342 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7343 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7344 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7346 /* WaSwitchSolVfFArbitrationPriority:hsw */
7347 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7349 lpt_init_clock_gating(dev_priv);
7352 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7356 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7358 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7359 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7360 I915_READ(ILK_DISPLAY_CHICKEN1) |
7363 /* WaDisableBackToBackFlipFix:ivb */
7364 I915_WRITE(IVB_CHICKEN3,
7365 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7366 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7368 if (IS_IVB_GT1(dev_priv))
7369 I915_WRITE(GEN7_ROW_CHICKEN2,
7370 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7372 /* must write both registers */
7373 I915_WRITE(GEN7_ROW_CHICKEN2,
7374 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7375 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7376 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7380 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7381 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7383 I915_WRITE(GEN6_UCGCTL2,
7384 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7386 /* This is required by WaCatErrorRejectionIssue:ivb */
7387 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7388 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7389 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7391 g4x_disable_trickle_feed(dev_priv);
7393 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7394 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7395 snpcr |= GEN6_MBC_SNPCR_MED;
7396 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7398 if (!HAS_PCH_NOP(dev_priv))
7399 cpt_init_clock_gating(dev_priv);
7401 gen6_check_mch_setup(dev_priv);
7404 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7406 /* WaDisableBackToBackFlipFix:vlv */
7407 I915_WRITE(IVB_CHICKEN3,
7408 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7409 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7411 /* WaDisableDopClockGating:vlv */
7412 I915_WRITE(GEN7_ROW_CHICKEN2,
7413 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7415 /* This is required by WaCatErrorRejectionIssue:vlv */
7416 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7417 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7418 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7421 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7422 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7424 I915_WRITE(GEN6_UCGCTL2,
7425 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7427 /* WaDisableL3Bank2xClockGate:vlv
7428 * Disabling L3 clock gating- MMIO 940c[25] = 1
7429 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7430 I915_WRITE(GEN7_UCGCTL4,
7431 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7434 * WaDisableVLVClockGating_VBIIssue:vlv
7435 * Disable clock gating on th GCFG unit to prevent a delay
7436 * in the reporting of vblank events.
7438 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7441 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7443 /* WaVSRefCountFullforceMissDisable:chv */
7444 /* WaDSRefCountFullforceMissDisable:chv */
7445 I915_WRITE(GEN7_FF_THREAD_MODE,
7446 I915_READ(GEN7_FF_THREAD_MODE) &
7447 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7449 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7450 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7451 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7453 /* WaDisableCSUnitClockGating:chv */
7454 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7455 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7457 /* WaDisableSDEUnitClockGating:chv */
7458 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7459 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7462 * WaProgramL3SqcReg1Default:chv
7463 * See gfxspecs/Related Documents/Performance Guide/
7464 * LSQC Setting Recommendations.
7466 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7469 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7473 I915_WRITE(RENCLK_GATE_D1, 0);
7474 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7475 GS_UNIT_CLOCK_GATE_DISABLE |
7476 CL_UNIT_CLOCK_GATE_DISABLE);
7477 I915_WRITE(RAMCLK_GATE_D, 0);
7478 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7479 OVRUNIT_CLOCK_GATE_DISABLE |
7480 OVCUNIT_CLOCK_GATE_DISABLE;
7481 if (IS_GM45(dev_priv))
7482 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7483 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7485 g4x_disable_trickle_feed(dev_priv);
7488 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7490 struct intel_uncore *uncore = &dev_priv->uncore;
7492 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7493 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7494 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7495 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7496 intel_uncore_write16(uncore, DEUC, 0);
7497 intel_uncore_write(uncore,
7499 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7502 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7504 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7505 I965_RCC_CLOCK_GATE_DISABLE |
7506 I965_RCPB_CLOCK_GATE_DISABLE |
7507 I965_ISC_CLOCK_GATE_DISABLE |
7508 I965_FBC_CLOCK_GATE_DISABLE);
7509 I915_WRITE(RENCLK_GATE_D2, 0);
7510 I915_WRITE(MI_ARB_STATE,
7511 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7514 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7516 u32 dstate = I915_READ(D_STATE);
7518 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7519 DSTATE_DOT_CLOCK_GATING;
7520 I915_WRITE(D_STATE, dstate);
7522 if (IS_PINEVIEW(dev_priv))
7523 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7525 /* IIR "flip pending" means done if this bit is set */
7526 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7528 /* interrupts should cause a wake up from C3 */
7529 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7531 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7532 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7534 I915_WRITE(MI_ARB_STATE,
7535 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7538 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7540 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7542 /* interrupts should cause a wake up from C3 */
7543 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7544 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7546 I915_WRITE(MEM_MODE,
7547 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7550 * Have FBC ignore 3D activity since we use software
7551 * render tracking, and otherwise a pure 3D workload
7552 * (even if it just renders a single frame and then does
7553 * abosultely nothing) would not allow FBC to recompress
7554 * until a 2D blit occurs.
7557 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7560 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7562 I915_WRITE(MEM_MODE,
7563 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7564 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7567 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7569 dev_priv->display.init_clock_gating(dev_priv);
7572 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7574 if (HAS_PCH_LPT(dev_priv))
7575 lpt_suspend_hw(dev_priv);
7578 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7580 drm_dbg_kms(&dev_priv->drm,
7581 "No clock gating settings or workarounds applied.\n");
7585 * intel_init_clock_gating_hooks - setup the clock gating hooks
7586 * @dev_priv: device private
7588 * Setup the hooks that configure which clocks of a given platform can be
7589 * gated and also apply various GT and display specific workarounds for these
7590 * platforms. Note that some GT specific workarounds are applied separately
7591 * when GPU contexts or batchbuffers start their execution.
7593 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7595 if (IS_GEN(dev_priv, 12))
7596 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7597 else if (IS_GEN(dev_priv, 11))
7598 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7599 else if (IS_CANNONLAKE(dev_priv))
7600 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7601 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7602 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7603 else if (IS_SKYLAKE(dev_priv))
7604 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7605 else if (IS_KABYLAKE(dev_priv))
7606 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7607 else if (IS_BROXTON(dev_priv))
7608 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7609 else if (IS_GEMINILAKE(dev_priv))
7610 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7611 else if (IS_BROADWELL(dev_priv))
7612 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7613 else if (IS_CHERRYVIEW(dev_priv))
7614 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7615 else if (IS_HASWELL(dev_priv))
7616 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7617 else if (IS_IVYBRIDGE(dev_priv))
7618 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7619 else if (IS_VALLEYVIEW(dev_priv))
7620 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7621 else if (IS_GEN(dev_priv, 6))
7622 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7623 else if (IS_GEN(dev_priv, 5))
7624 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7625 else if (IS_G4X(dev_priv))
7626 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7627 else if (IS_I965GM(dev_priv))
7628 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7629 else if (IS_I965G(dev_priv))
7630 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7631 else if (IS_GEN(dev_priv, 3))
7632 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7633 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7634 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7635 else if (IS_GEN(dev_priv, 2))
7636 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7638 MISSING_CASE(INTEL_DEVID(dev_priv));
7639 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7643 /* Set up chip specific power management-related functions */
7644 void intel_init_pm(struct drm_i915_private *dev_priv)
7647 if (IS_PINEVIEW(dev_priv))
7648 pnv_get_mem_freq(dev_priv);
7649 else if (IS_GEN(dev_priv, 5))
7650 ilk_get_mem_freq(dev_priv);
7652 if (intel_has_sagv(dev_priv))
7653 skl_setup_sagv_block_time(dev_priv);
7655 /* For FIFO watermark updates */
7656 if (INTEL_GEN(dev_priv) >= 9) {
7657 skl_setup_wm_latency(dev_priv);
7658 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7659 } else if (HAS_PCH_SPLIT(dev_priv)) {
7660 ilk_setup_wm_latency(dev_priv);
7662 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7663 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7664 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7665 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7666 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7667 dev_priv->display.compute_intermediate_wm =
7668 ilk_compute_intermediate_wm;
7669 dev_priv->display.initial_watermarks =
7670 ilk_initial_watermarks;
7671 dev_priv->display.optimize_watermarks =
7672 ilk_optimize_watermarks;
7674 drm_dbg_kms(&dev_priv->drm,
7675 "Failed to read display plane latency. "
7678 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7679 vlv_setup_wm_latency(dev_priv);
7680 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7681 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7682 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7683 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7684 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7685 } else if (IS_G4X(dev_priv)) {
7686 g4x_setup_wm_latency(dev_priv);
7687 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7688 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7689 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7690 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7691 } else if (IS_PINEVIEW(dev_priv)) {
7692 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7695 dev_priv->mem_freq)) {
7696 drm_info(&dev_priv->drm,
7697 "failed to find known CxSR latency "
7698 "(found ddr%s fsb freq %d, mem freq %d), "
7700 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7701 dev_priv->fsb_freq, dev_priv->mem_freq);
7702 /* Disable CxSR and never update its watermark again */
7703 intel_set_memory_cxsr(dev_priv, false);
7704 dev_priv->display.update_wm = NULL;
7706 dev_priv->display.update_wm = pnv_update_wm;
7707 } else if (IS_GEN(dev_priv, 4)) {
7708 dev_priv->display.update_wm = i965_update_wm;
7709 } else if (IS_GEN(dev_priv, 3)) {
7710 dev_priv->display.update_wm = i9xx_update_wm;
7711 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7712 } else if (IS_GEN(dev_priv, 2)) {
7713 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7714 dev_priv->display.update_wm = i845_update_wm;
7715 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7717 dev_priv->display.update_wm = i9xx_update_wm;
7718 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7721 drm_err(&dev_priv->drm,
7722 "unexpected fall-through in %s\n", __func__);
7726 void intel_pm_setup(struct drm_i915_private *dev_priv)
7728 dev_priv->runtime_pm.suspended = false;
7729 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7732 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7734 struct intel_dbuf_state *dbuf_state;
7736 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7740 return &dbuf_state->base;
7743 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7744 struct intel_global_state *state)
7749 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7750 .atomic_duplicate_state = intel_dbuf_duplicate_state,
7751 .atomic_destroy_state = intel_dbuf_destroy_state,
7754 struct intel_dbuf_state *
7755 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7757 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7758 struct intel_global_state *dbuf_state;
7760 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7761 if (IS_ERR(dbuf_state))
7762 return ERR_CAST(dbuf_state);
7764 return to_intel_dbuf_state(dbuf_state);
7767 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7769 struct intel_dbuf_state *dbuf_state;
7771 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7775 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7776 &dbuf_state->base, &intel_dbuf_funcs);
7781 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7783 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7784 const struct intel_dbuf_state *new_dbuf_state =
7785 intel_atomic_get_new_dbuf_state(state);
7786 const struct intel_dbuf_state *old_dbuf_state =
7787 intel_atomic_get_old_dbuf_state(state);
7789 if (!new_dbuf_state ||
7790 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7793 WARN_ON(!new_dbuf_state->base.changed);
7795 gen9_dbuf_slices_update(dev_priv,
7796 old_dbuf_state->enabled_slices |
7797 new_dbuf_state->enabled_slices);
7800 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7802 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7803 const struct intel_dbuf_state *new_dbuf_state =
7804 intel_atomic_get_new_dbuf_state(state);
7805 const struct intel_dbuf_state *old_dbuf_state =
7806 intel_atomic_get_old_dbuf_state(state);
7808 if (!new_dbuf_state ||
7809 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7812 WARN_ON(!new_dbuf_state->base.changed);
7814 gen9_dbuf_slices_update(dev_priv,
7815 new_dbuf_state->enabled_slices);