2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
35 #include "display/intel_atomic.h"
36 #include "display/intel_bw.h"
37 #include "display/intel_display_types.h"
38 #include "display/intel_fbc.h"
39 #include "display/intel_sprite.h"
41 #include "gt/intel_llc.h"
44 #include "i915_fixed.h"
46 #include "i915_trace.h"
48 #include "intel_sideband.h"
49 #include "../../../platform/x86/intel_ips.h"
51 /* Stores plane specific WM parameters */
52 struct skl_wm_params {
53 bool x_tiled, y_tiled;
60 u32 plane_bytes_per_line;
61 uint_fixed_16_16_t plane_blocks_per_line;
62 uint_fixed_16_16_t y_tile_minimum;
67 /* used in computing the new watermarks state */
68 struct intel_wm_config {
69 unsigned int num_pipes_active;
74 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
76 if (HAS_LLC(dev_priv)) {
78 * WaCompressedResourceDisplayNewHashMode:skl,kbl
79 * Display WA #0390: skl,kbl
81 * Must match Sampler, Pixel Back End, and Media. See
82 * WaCompressedResourceSamplerPbeMediaNewHashMode.
84 I915_WRITE(CHICKEN_PAR1_1,
85 I915_READ(CHICKEN_PAR1_1) |
86 SKL_DE_COMPRESSED_HASH_MODE);
89 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
90 I915_WRITE(CHICKEN_PAR1_1,
91 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
93 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
94 I915_WRITE(GEN8_CHICKEN_DCPR_1,
95 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
98 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
99 * Display WA #0859: skl,bxt,kbl,glk,cfl
101 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
102 DISP_FBC_MEMORY_WAKE);
105 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
107 gen9_init_clock_gating(dev_priv);
109 /* WaDisableSDEUnitClockGating:bxt */
110 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
111 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
115 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
117 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
118 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
121 * Wa: Backlight PWM may stop in the asserted state, causing backlight
124 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
125 PWM1_GATING_DIS | PWM2_GATING_DIS);
128 * Lower the display internal timeout.
129 * This is needed to avoid any hard hangs when DSI port PLL
130 * is off and a MMIO access is attempted by any privilege
131 * application, using batch buffers or any other means.
133 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
136 * WaFbcTurnOffFbcWatermark:bxt
137 * Display WA #0562: bxt
139 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
143 * WaFbcHighMemBwCorruptionAvoidance:bxt
144 * Display WA #0883: bxt
146 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
147 ILK_DPFC_DISABLE_DUMMY0);
150 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
152 gen9_init_clock_gating(dev_priv);
155 * WaDisablePWMClockGating:glk
156 * Backlight PWM may stop in the asserted state, causing backlight
159 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
160 PWM1_GATING_DIS | PWM2_GATING_DIS);
163 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
167 tmp = I915_READ(CLKCFG);
169 switch (tmp & CLKCFG_FSB_MASK) {
171 dev_priv->fsb_freq = 533; /* 133*4 */
174 dev_priv->fsb_freq = 800; /* 200*4 */
177 dev_priv->fsb_freq = 667; /* 167*4 */
180 dev_priv->fsb_freq = 400; /* 100*4 */
184 switch (tmp & CLKCFG_MEM_MASK) {
186 dev_priv->mem_freq = 533;
189 dev_priv->mem_freq = 667;
192 dev_priv->mem_freq = 800;
196 /* detect pineview DDR3 setting */
197 tmp = I915_READ(CSHRDDR3CTL);
198 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
201 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
205 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
206 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
208 switch (ddrpll & 0xff) {
210 dev_priv->mem_freq = 800;
213 dev_priv->mem_freq = 1066;
216 dev_priv->mem_freq = 1333;
219 dev_priv->mem_freq = 1600;
222 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
224 dev_priv->mem_freq = 0;
228 switch (csipll & 0x3ff) {
230 dev_priv->fsb_freq = 3200;
233 dev_priv->fsb_freq = 3733;
236 dev_priv->fsb_freq = 4266;
239 dev_priv->fsb_freq = 4800;
242 dev_priv->fsb_freq = 5333;
245 dev_priv->fsb_freq = 5866;
248 dev_priv->fsb_freq = 6400;
251 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
253 dev_priv->fsb_freq = 0;
258 static const struct cxsr_latency cxsr_latency_table[] = {
259 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
260 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
261 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
262 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
263 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
265 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
266 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
267 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
268 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
269 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
271 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
272 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
273 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
274 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
275 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
277 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
278 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
279 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
280 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
281 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
283 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
284 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
285 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
286 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
287 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
289 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
290 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
291 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
292 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
293 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
296 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
301 const struct cxsr_latency *latency;
304 if (fsb == 0 || mem == 0)
307 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
308 latency = &cxsr_latency_table[i];
309 if (is_desktop == latency->is_desktop &&
310 is_ddr3 == latency->is_ddr3 &&
311 fsb == latency->fsb_freq && mem == latency->mem_freq)
315 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
320 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
324 vlv_punit_get(dev_priv);
326 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
328 val &= ~FORCE_DDR_HIGH_FREQ;
330 val |= FORCE_DDR_HIGH_FREQ;
331 val &= ~FORCE_DDR_LOW_FREQ;
332 val |= FORCE_DDR_FREQ_REQ_ACK;
333 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
335 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
336 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
337 drm_err(&dev_priv->drm,
338 "timed out waiting for Punit DDR DVFS request\n");
340 vlv_punit_put(dev_priv);
343 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
347 vlv_punit_get(dev_priv);
349 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
351 val |= DSP_MAXFIFO_PM5_ENABLE;
353 val &= ~DSP_MAXFIFO_PM5_ENABLE;
354 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
356 vlv_punit_put(dev_priv);
359 #define FW_WM(value, plane) \
360 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
362 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
367 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
368 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
369 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
370 POSTING_READ(FW_BLC_SELF_VLV);
371 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
372 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
373 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
374 POSTING_READ(FW_BLC_SELF);
375 } else if (IS_PINEVIEW(dev_priv)) {
376 val = I915_READ(DSPFW3);
377 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
379 val |= PINEVIEW_SELF_REFRESH_EN;
381 val &= ~PINEVIEW_SELF_REFRESH_EN;
382 I915_WRITE(DSPFW3, val);
383 POSTING_READ(DSPFW3);
384 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
385 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
386 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
387 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
388 I915_WRITE(FW_BLC_SELF, val);
389 POSTING_READ(FW_BLC_SELF);
390 } else if (IS_I915GM(dev_priv)) {
392 * FIXME can't find a bit like this for 915G, and
393 * and yet it does have the related watermark in
394 * FW_BLC_SELF. What's going on?
396 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
397 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
398 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
399 I915_WRITE(INSTPM, val);
400 POSTING_READ(INSTPM);
405 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
407 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
408 enableddisabled(enable),
409 enableddisabled(was_enabled));
415 * intel_set_memory_cxsr - Configure CxSR state
416 * @dev_priv: i915 device
417 * @enable: Allow vs. disallow CxSR
419 * Allow or disallow the system to enter a special CxSR
420 * (C-state self refresh) state. What typically happens in CxSR mode
421 * is that several display FIFOs may get combined into a single larger
422 * FIFO for a particular plane (so called max FIFO mode) to allow the
423 * system to defer memory fetches longer, and the memory will enter
426 * Note that enabling CxSR does not guarantee that the system enter
427 * this special mode, nor does it guarantee that the system stays
428 * in that mode once entered. So this just allows/disallows the system
429 * to autonomously utilize the CxSR mode. Other factors such as core
430 * C-states will affect when/if the system actually enters/exits the
433 * Note that on VLV/CHV this actually only controls the max FIFO mode,
434 * and the system is free to enter/exit memory self refresh at any time
435 * even when the use of CxSR has been disallowed.
437 * While the system is actually in the CxSR/max FIFO mode, some plane
438 * control registers will not get latched on vblank. Thus in order to
439 * guarantee the system will respond to changes in the plane registers
440 * we must always disallow CxSR prior to making changes to those registers.
441 * Unfortunately the system will re-evaluate the CxSR conditions at
442 * frame start which happens after vblank start (which is when the plane
443 * registers would get latched), so we can't proceed with the plane update
444 * during the same frame where we disallowed CxSR.
446 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
447 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
448 * the hardware w.r.t. HPLL SR when writing to plane registers.
449 * Disallowing just CxSR is sufficient.
451 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
455 mutex_lock(&dev_priv->wm.wm_mutex);
456 ret = _intel_set_memory_cxsr(dev_priv, enable);
457 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
458 dev_priv->wm.vlv.cxsr = enable;
459 else if (IS_G4X(dev_priv))
460 dev_priv->wm.g4x.cxsr = enable;
461 mutex_unlock(&dev_priv->wm.wm_mutex);
467 * Latency for FIFO fetches is dependent on several factors:
468 * - memory configuration (speed, channels)
470 * - current MCH state
471 * It can be fairly high in some situations, so here we assume a fairly
472 * pessimal value. It's a tradeoff between extra memory fetches (if we
473 * set this value too high, the FIFO will fetch frequently to stay full)
474 * and power consumption (set it too low to save power and we might see
475 * FIFO underruns and display "flicker").
477 * A value of 5us seems to be a good balance; safe for very low end
478 * platforms but not overly aggressive on lower latency configs.
480 static const int pessimal_latency_ns = 5000;
482 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
483 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
485 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
487 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
488 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
489 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
490 enum pipe pipe = crtc->pipe;
491 int sprite0_start, sprite1_start;
492 u32 dsparb, dsparb2, dsparb3;
496 dsparb = I915_READ(DSPARB);
497 dsparb2 = I915_READ(DSPARB2);
498 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
499 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
502 dsparb = I915_READ(DSPARB);
503 dsparb2 = I915_READ(DSPARB2);
504 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
505 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
508 dsparb2 = I915_READ(DSPARB2);
509 dsparb3 = I915_READ(DSPARB3);
510 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
511 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
518 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
519 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
520 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
521 fifo_state->plane[PLANE_CURSOR] = 63;
524 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
525 enum i9xx_plane_id i9xx_plane)
527 u32 dsparb = I915_READ(DSPARB);
530 size = dsparb & 0x7f;
531 if (i9xx_plane == PLANE_B)
532 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
534 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
535 dsparb, plane_name(i9xx_plane), size);
540 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
541 enum i9xx_plane_id i9xx_plane)
543 u32 dsparb = I915_READ(DSPARB);
546 size = dsparb & 0x1ff;
547 if (i9xx_plane == PLANE_B)
548 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
549 size >>= 1; /* Convert to cachelines */
551 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
552 dsparb, plane_name(i9xx_plane), size);
557 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
558 enum i9xx_plane_id i9xx_plane)
560 u32 dsparb = I915_READ(DSPARB);
563 size = dsparb & 0x7f;
564 size >>= 2; /* Convert to cachelines */
566 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
567 dsparb, plane_name(i9xx_plane), size);
572 /* Pineview has different values for various configs */
573 static const struct intel_watermark_params pnv_display_wm = {
574 .fifo_size = PINEVIEW_DISPLAY_FIFO,
575 .max_wm = PINEVIEW_MAX_WM,
576 .default_wm = PINEVIEW_DFT_WM,
577 .guard_size = PINEVIEW_GUARD_WM,
578 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
581 static const struct intel_watermark_params pnv_display_hplloff_wm = {
582 .fifo_size = PINEVIEW_DISPLAY_FIFO,
583 .max_wm = PINEVIEW_MAX_WM,
584 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
585 .guard_size = PINEVIEW_GUARD_WM,
586 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
589 static const struct intel_watermark_params pnv_cursor_wm = {
590 .fifo_size = PINEVIEW_CURSOR_FIFO,
591 .max_wm = PINEVIEW_CURSOR_MAX_WM,
592 .default_wm = PINEVIEW_CURSOR_DFT_WM,
593 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
594 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
597 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
598 .fifo_size = PINEVIEW_CURSOR_FIFO,
599 .max_wm = PINEVIEW_CURSOR_MAX_WM,
600 .default_wm = PINEVIEW_CURSOR_DFT_WM,
601 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
602 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
605 static const struct intel_watermark_params i965_cursor_wm_info = {
606 .fifo_size = I965_CURSOR_FIFO,
607 .max_wm = I965_CURSOR_MAX_WM,
608 .default_wm = I965_CURSOR_DFT_WM,
610 .cacheline_size = I915_FIFO_LINE_SIZE,
613 static const struct intel_watermark_params i945_wm_info = {
614 .fifo_size = I945_FIFO_SIZE,
615 .max_wm = I915_MAX_WM,
618 .cacheline_size = I915_FIFO_LINE_SIZE,
621 static const struct intel_watermark_params i915_wm_info = {
622 .fifo_size = I915_FIFO_SIZE,
623 .max_wm = I915_MAX_WM,
626 .cacheline_size = I915_FIFO_LINE_SIZE,
629 static const struct intel_watermark_params i830_a_wm_info = {
630 .fifo_size = I855GM_FIFO_SIZE,
631 .max_wm = I915_MAX_WM,
634 .cacheline_size = I830_FIFO_LINE_SIZE,
637 static const struct intel_watermark_params i830_bc_wm_info = {
638 .fifo_size = I855GM_FIFO_SIZE,
639 .max_wm = I915_MAX_WM/2,
642 .cacheline_size = I830_FIFO_LINE_SIZE,
645 static const struct intel_watermark_params i845_wm_info = {
646 .fifo_size = I830_FIFO_SIZE,
647 .max_wm = I915_MAX_WM,
650 .cacheline_size = I830_FIFO_LINE_SIZE,
654 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
655 * @pixel_rate: Pipe pixel rate in kHz
656 * @cpp: Plane bytes per pixel
657 * @latency: Memory wakeup latency in 0.1us units
659 * Compute the watermark using the method 1 or "small buffer"
660 * formula. The caller may additonally add extra cachelines
661 * to account for TLB misses and clock crossings.
663 * This method is concerned with the short term drain rate
664 * of the FIFO, ie. it does not account for blanking periods
665 * which would effectively reduce the average drain rate across
666 * a longer period. The name "small" refers to the fact the
667 * FIFO is relatively small compared to the amount of data
670 * The FIFO level vs. time graph might look something like:
674 * __---__---__ (- plane active, _ blanking)
677 * or perhaps like this:
680 * __----__----__ (- plane active, _ blanking)
684 * The watermark in bytes
686 static unsigned int intel_wm_method1(unsigned int pixel_rate,
688 unsigned int latency)
692 ret = mul_u32_u32(pixel_rate, cpp * latency);
693 ret = DIV_ROUND_UP_ULL(ret, 10000);
699 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
700 * @pixel_rate: Pipe pixel rate in kHz
701 * @htotal: Pipe horizontal total
702 * @width: Plane width in pixels
703 * @cpp: Plane bytes per pixel
704 * @latency: Memory wakeup latency in 0.1us units
706 * Compute the watermark using the method 2 or "large buffer"
707 * formula. The caller may additonally add extra cachelines
708 * to account for TLB misses and clock crossings.
710 * This method is concerned with the long term drain rate
711 * of the FIFO, ie. it does account for blanking periods
712 * which effectively reduce the average drain rate across
713 * a longer period. The name "large" refers to the fact the
714 * FIFO is relatively large compared to the amount of data
717 * The FIFO level vs. time graph might look something like:
722 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
726 * The watermark in bytes
728 static unsigned int intel_wm_method2(unsigned int pixel_rate,
732 unsigned int latency)
737 * FIXME remove once all users are computing
738 * watermarks in the correct place.
740 if (WARN_ON_ONCE(htotal == 0))
743 ret = (latency * pixel_rate) / (htotal * 10000);
744 ret = (ret + 1) * width * cpp;
750 * intel_calculate_wm - calculate watermark level
751 * @pixel_rate: pixel clock
752 * @wm: chip FIFO params
753 * @fifo_size: size of the FIFO buffer
754 * @cpp: bytes per pixel
755 * @latency_ns: memory latency for the platform
757 * Calculate the watermark level (the level at which the display plane will
758 * start fetching from memory again). Each chip has a different display
759 * FIFO size and allocation, so the caller needs to figure that out and pass
760 * in the correct intel_watermark_params structure.
762 * As the pixel clock runs, the FIFO will be drained at a rate that depends
763 * on the pixel size. When it reaches the watermark level, it'll start
764 * fetching FIFO line sized based chunks from memory until the FIFO fills
765 * past the watermark point. If the FIFO drains completely, a FIFO underrun
766 * will occur, and a display engine hang could result.
768 static unsigned int intel_calculate_wm(int pixel_rate,
769 const struct intel_watermark_params *wm,
770 int fifo_size, int cpp,
771 unsigned int latency_ns)
773 int entries, wm_size;
776 * Note: we need to make sure we don't overflow for various clock &
778 * clocks go from a few thousand to several hundred thousand.
779 * latency is usually a few thousand
781 entries = intel_wm_method1(pixel_rate, cpp,
783 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
785 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
787 wm_size = fifo_size - entries;
788 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
790 /* Don't promote wm_size to unsigned... */
791 if (wm_size > wm->max_wm)
792 wm_size = wm->max_wm;
794 wm_size = wm->default_wm;
797 * Bspec seems to indicate that the value shouldn't be lower than
798 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
799 * Lets go for 8 which is the burst size since certain platforms
800 * already use a hardcoded 8 (which is what the spec says should be
809 static bool is_disabling(int old, int new, int threshold)
811 return old >= threshold && new < threshold;
814 static bool is_enabling(int old, int new, int threshold)
816 return old < threshold && new >= threshold;
819 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
821 return dev_priv->wm.max_level + 1;
824 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
825 const struct intel_plane_state *plane_state)
827 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
829 /* FIXME check the 'enable' instead */
830 if (!crtc_state->hw.active)
834 * Treat cursor with fb as always visible since cursor updates
835 * can happen faster than the vrefresh rate, and the current
836 * watermark code doesn't handle that correctly. Cursor updates
837 * which set/clear the fb or change the cursor size are going
838 * to get throttled by intel_legacy_cursor_update() to work
839 * around this problem with the watermark code.
841 if (plane->id == PLANE_CURSOR)
842 return plane_state->hw.fb != NULL;
844 return plane_state->uapi.visible;
847 static bool intel_crtc_active(struct intel_crtc *crtc)
849 /* Be paranoid as we can arrive here with only partial
850 * state retrieved from the hardware during setup.
852 * We can ditch the adjusted_mode.crtc_clock check as soon
853 * as Haswell has gained clock readout/fastboot support.
855 * We can ditch the crtc->primary->state->fb check as soon as we can
856 * properly reconstruct framebuffers.
858 * FIXME: The intel_crtc->active here should be switched to
859 * crtc->state->active once we have proper CRTC states wired up
862 return crtc->active && crtc->base.primary->state->fb &&
863 crtc->config->hw.adjusted_mode.crtc_clock;
866 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
868 struct intel_crtc *crtc, *enabled = NULL;
870 for_each_intel_crtc(&dev_priv->drm, crtc) {
871 if (intel_crtc_active(crtc)) {
881 static void pnv_update_wm(struct intel_crtc *unused_crtc)
883 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
884 struct intel_crtc *crtc;
885 const struct cxsr_latency *latency;
889 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
894 drm_dbg_kms(&dev_priv->drm,
895 "Unknown FSB/MEM found, disable CxSR\n");
896 intel_set_memory_cxsr(dev_priv, false);
900 crtc = single_enabled_crtc(dev_priv);
902 const struct drm_display_mode *adjusted_mode =
903 &crtc->config->hw.adjusted_mode;
904 const struct drm_framebuffer *fb =
905 crtc->base.primary->state->fb;
906 int cpp = fb->format->cpp[0];
907 int clock = adjusted_mode->crtc_clock;
910 wm = intel_calculate_wm(clock, &pnv_display_wm,
911 pnv_display_wm.fifo_size,
912 cpp, latency->display_sr);
913 reg = I915_READ(DSPFW1);
914 reg &= ~DSPFW_SR_MASK;
915 reg |= FW_WM(wm, SR);
916 I915_WRITE(DSPFW1, reg);
917 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
920 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
921 pnv_display_wm.fifo_size,
922 4, latency->cursor_sr);
923 reg = I915_READ(DSPFW3);
924 reg &= ~DSPFW_CURSOR_SR_MASK;
925 reg |= FW_WM(wm, CURSOR_SR);
926 I915_WRITE(DSPFW3, reg);
928 /* Display HPLL off SR */
929 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
930 pnv_display_hplloff_wm.fifo_size,
931 cpp, latency->display_hpll_disable);
932 reg = I915_READ(DSPFW3);
933 reg &= ~DSPFW_HPLL_SR_MASK;
934 reg |= FW_WM(wm, HPLL_SR);
935 I915_WRITE(DSPFW3, reg);
937 /* cursor HPLL off SR */
938 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
939 pnv_display_hplloff_wm.fifo_size,
940 4, latency->cursor_hpll_disable);
941 reg = I915_READ(DSPFW3);
942 reg &= ~DSPFW_HPLL_CURSOR_MASK;
943 reg |= FW_WM(wm, HPLL_CURSOR);
944 I915_WRITE(DSPFW3, reg);
945 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
947 intel_set_memory_cxsr(dev_priv, true);
949 intel_set_memory_cxsr(dev_priv, false);
954 * Documentation says:
955 * "If the line size is small, the TLB fetches can get in the way of the
956 * data fetches, causing some lag in the pixel data return which is not
957 * accounted for in the above formulas. The following adjustment only
958 * needs to be applied if eight whole lines fit in the buffer at once.
959 * The WM is adjusted upwards by the difference between the FIFO size
960 * and the size of 8 whole lines. This adjustment is always performed
961 * in the actual pixel depth regardless of whether FBC is enabled or not."
963 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
965 int tlb_miss = fifo_size * 64 - width * cpp * 8;
967 return max(0, tlb_miss);
970 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
971 const struct g4x_wm_values *wm)
975 for_each_pipe(dev_priv, pipe)
976 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
979 FW_WM(wm->sr.plane, SR) |
980 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
981 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
982 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
984 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
985 FW_WM(wm->sr.fbc, FBC_SR) |
986 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
987 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
988 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
989 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
991 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
992 FW_WM(wm->sr.cursor, CURSOR_SR) |
993 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
994 FW_WM(wm->hpll.plane, HPLL_SR));
996 POSTING_READ(DSPFW1);
999 #define FW_WM_VLV(value, plane) \
1000 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1002 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1003 const struct vlv_wm_values *wm)
1007 for_each_pipe(dev_priv, pipe) {
1008 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1010 I915_WRITE(VLV_DDL(pipe),
1011 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1012 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1013 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1014 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1018 * Zero the (unused) WM1 watermarks, and also clear all the
1019 * high order bits so that there are no out of bounds values
1020 * present in the registers during the reprogramming.
1022 I915_WRITE(DSPHOWM, 0);
1023 I915_WRITE(DSPHOWM1, 0);
1024 I915_WRITE(DSPFW4, 0);
1025 I915_WRITE(DSPFW5, 0);
1026 I915_WRITE(DSPFW6, 0);
1029 FW_WM(wm->sr.plane, SR) |
1030 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1031 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1032 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1034 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1035 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1036 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1038 FW_WM(wm->sr.cursor, CURSOR_SR));
1040 if (IS_CHERRYVIEW(dev_priv)) {
1041 I915_WRITE(DSPFW7_CHV,
1042 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1043 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1044 I915_WRITE(DSPFW8_CHV,
1045 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1046 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1047 I915_WRITE(DSPFW9_CHV,
1048 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1049 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1051 FW_WM(wm->sr.plane >> 9, SR_HI) |
1052 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1053 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1054 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1055 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1056 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1057 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1058 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1059 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1060 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1063 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1064 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1066 FW_WM(wm->sr.plane >> 9, SR_HI) |
1067 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1068 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1069 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1070 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1071 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1072 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1075 POSTING_READ(DSPFW1);
1080 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1082 /* all latencies in usec */
1083 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1084 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1085 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1087 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1090 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1093 * DSPCNTR[13] supposedly controls whether the
1094 * primary plane can use the FIFO space otherwise
1095 * reserved for the sprite plane. It's not 100% clear
1096 * what the actual FIFO size is, but it looks like we
1097 * can happily set both primary and sprite watermarks
1098 * up to 127 cachelines. So that would seem to mean
1099 * that either DSPCNTR[13] doesn't do anything, or that
1100 * the total FIFO is >= 256 cachelines in size. Either
1101 * way, we don't seem to have to worry about this
1102 * repartitioning as the maximum watermark value the
1103 * register can hold for each plane is lower than the
1104 * minimum FIFO size.
1110 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1112 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1114 MISSING_CASE(plane_id);
1119 static int g4x_fbc_fifo_size(int level)
1122 case G4X_WM_LEVEL_SR:
1124 case G4X_WM_LEVEL_HPLL:
1127 MISSING_CASE(level);
1132 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1133 const struct intel_plane_state *plane_state,
1136 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1137 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1138 const struct drm_display_mode *adjusted_mode =
1139 &crtc_state->hw.adjusted_mode;
1140 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1141 unsigned int clock, htotal, cpp, width, wm;
1146 if (!intel_wm_plane_visible(crtc_state, plane_state))
1149 cpp = plane_state->hw.fb->format->cpp[0];
1152 * Not 100% sure which way ELK should go here as the
1153 * spec only says CL/CTG should assume 32bpp and BW
1154 * doesn't need to. But as these things followed the
1155 * mobile vs. desktop lines on gen3 as well, let's
1156 * assume ELK doesn't need this.
1158 * The spec also fails to list such a restriction for
1159 * the HPLL watermark, which seems a little strange.
1160 * Let's use 32bpp for the HPLL watermark as well.
1162 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1163 level != G4X_WM_LEVEL_NORMAL)
1166 clock = adjusted_mode->crtc_clock;
1167 htotal = adjusted_mode->crtc_htotal;
1169 width = drm_rect_width(&plane_state->uapi.dst);
1171 if (plane->id == PLANE_CURSOR) {
1172 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1173 } else if (plane->id == PLANE_PRIMARY &&
1174 level == G4X_WM_LEVEL_NORMAL) {
1175 wm = intel_wm_method1(clock, cpp, latency);
1177 unsigned int small, large;
1179 small = intel_wm_method1(clock, cpp, latency);
1180 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1182 wm = min(small, large);
1185 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1188 wm = DIV_ROUND_UP(wm, 64) + 2;
1190 return min_t(unsigned int, wm, USHRT_MAX);
1193 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1194 int level, enum plane_id plane_id, u16 value)
1196 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1199 for (; level < intel_wm_num_levels(dev_priv); level++) {
1200 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1202 dirty |= raw->plane[plane_id] != value;
1203 raw->plane[plane_id] = value;
1209 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1210 int level, u16 value)
1212 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1215 /* NORMAL level doesn't have an FBC watermark */
1216 level = max(level, G4X_WM_LEVEL_SR);
1218 for (; level < intel_wm_num_levels(dev_priv); level++) {
1219 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1221 dirty |= raw->fbc != value;
1228 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1229 const struct intel_plane_state *plane_state,
1232 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1233 const struct intel_plane_state *plane_state)
1235 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1236 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1237 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1238 enum plane_id plane_id = plane->id;
1242 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1243 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1244 if (plane_id == PLANE_PRIMARY)
1245 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1249 for (level = 0; level < num_levels; level++) {
1250 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1253 wm = g4x_compute_wm(crtc_state, plane_state, level);
1254 max_wm = g4x_plane_fifo_size(plane_id, level);
1259 dirty |= raw->plane[plane_id] != wm;
1260 raw->plane[plane_id] = wm;
1262 if (plane_id != PLANE_PRIMARY ||
1263 level == G4X_WM_LEVEL_NORMAL)
1266 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1267 raw->plane[plane_id]);
1268 max_wm = g4x_fbc_fifo_size(level);
1271 * FBC wm is not mandatory as we
1272 * can always just disable its use.
1277 dirty |= raw->fbc != wm;
1281 /* mark watermarks as invalid */
1282 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1284 if (plane_id == PLANE_PRIMARY)
1285 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1289 drm_dbg_kms(&dev_priv->drm,
1290 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1292 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1293 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1294 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1296 if (plane_id == PLANE_PRIMARY)
1297 drm_dbg_kms(&dev_priv->drm,
1298 "FBC watermarks: SR=%d, HPLL=%d\n",
1299 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1300 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1306 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1307 enum plane_id plane_id, int level)
1309 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1311 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1314 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1317 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1319 if (level > dev_priv->wm.max_level)
1322 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1323 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1324 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1327 /* mark all levels starting from 'level' as invalid */
1328 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1329 struct g4x_wm_state *wm_state, int level)
1331 if (level <= G4X_WM_LEVEL_NORMAL) {
1332 enum plane_id plane_id;
1334 for_each_plane_id_on_crtc(crtc, plane_id)
1335 wm_state->wm.plane[plane_id] = USHRT_MAX;
1338 if (level <= G4X_WM_LEVEL_SR) {
1339 wm_state->cxsr = false;
1340 wm_state->sr.cursor = USHRT_MAX;
1341 wm_state->sr.plane = USHRT_MAX;
1342 wm_state->sr.fbc = USHRT_MAX;
1345 if (level <= G4X_WM_LEVEL_HPLL) {
1346 wm_state->hpll_en = false;
1347 wm_state->hpll.cursor = USHRT_MAX;
1348 wm_state->hpll.plane = USHRT_MAX;
1349 wm_state->hpll.fbc = USHRT_MAX;
1353 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1356 if (level < G4X_WM_LEVEL_SR)
1359 if (level >= G4X_WM_LEVEL_SR &&
1360 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1363 if (level >= G4X_WM_LEVEL_HPLL &&
1364 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1370 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1372 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1373 struct intel_atomic_state *state =
1374 to_intel_atomic_state(crtc_state->uapi.state);
1375 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1376 int num_active_planes = hweight8(crtc_state->active_planes &
1377 ~BIT(PLANE_CURSOR));
1378 const struct g4x_pipe_wm *raw;
1379 const struct intel_plane_state *old_plane_state;
1380 const struct intel_plane_state *new_plane_state;
1381 struct intel_plane *plane;
1382 enum plane_id plane_id;
1384 unsigned int dirty = 0;
1386 for_each_oldnew_intel_plane_in_state(state, plane,
1388 new_plane_state, i) {
1389 if (new_plane_state->hw.crtc != &crtc->base &&
1390 old_plane_state->hw.crtc != &crtc->base)
1393 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1394 dirty |= BIT(plane->id);
1400 level = G4X_WM_LEVEL_NORMAL;
1401 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1404 raw = &crtc_state->wm.g4x.raw[level];
1405 for_each_plane_id_on_crtc(crtc, plane_id)
1406 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1408 level = G4X_WM_LEVEL_SR;
1409 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1412 raw = &crtc_state->wm.g4x.raw[level];
1413 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1414 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1415 wm_state->sr.fbc = raw->fbc;
1417 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1419 level = G4X_WM_LEVEL_HPLL;
1420 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1423 raw = &crtc_state->wm.g4x.raw[level];
1424 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1425 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1426 wm_state->hpll.fbc = raw->fbc;
1428 wm_state->hpll_en = wm_state->cxsr;
1433 if (level == G4X_WM_LEVEL_NORMAL)
1436 /* invalidate the higher levels */
1437 g4x_invalidate_wms(crtc, wm_state, level);
1440 * Determine if the FBC watermark(s) can be used. IF
1441 * this isn't the case we prefer to disable the FBC
1442 * watermark(s) rather than disable the SR/HPLL
1443 * level(s) entirely. 'level-1' is the highest valid
1446 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1451 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1453 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1454 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1455 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1456 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1457 struct intel_atomic_state *intel_state =
1458 to_intel_atomic_state(new_crtc_state->uapi.state);
1459 const struct intel_crtc_state *old_crtc_state =
1460 intel_atomic_get_old_crtc_state(intel_state, crtc);
1461 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1462 enum plane_id plane_id;
1464 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1465 *intermediate = *optimal;
1467 intermediate->cxsr = false;
1468 intermediate->hpll_en = false;
1472 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1473 !new_crtc_state->disable_cxsr;
1474 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1475 !new_crtc_state->disable_cxsr;
1476 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1478 for_each_plane_id_on_crtc(crtc, plane_id) {
1479 intermediate->wm.plane[plane_id] =
1480 max(optimal->wm.plane[plane_id],
1481 active->wm.plane[plane_id]);
1483 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1484 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1487 intermediate->sr.plane = max(optimal->sr.plane,
1489 intermediate->sr.cursor = max(optimal->sr.cursor,
1491 intermediate->sr.fbc = max(optimal->sr.fbc,
1494 intermediate->hpll.plane = max(optimal->hpll.plane,
1495 active->hpll.plane);
1496 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1497 active->hpll.cursor);
1498 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1501 drm_WARN_ON(&dev_priv->drm,
1502 (intermediate->sr.plane >
1503 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1504 intermediate->sr.cursor >
1505 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1506 intermediate->cxsr);
1507 drm_WARN_ON(&dev_priv->drm,
1508 (intermediate->sr.plane >
1509 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1510 intermediate->sr.cursor >
1511 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1512 intermediate->hpll_en);
1514 drm_WARN_ON(&dev_priv->drm,
1515 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1516 intermediate->fbc_en && intermediate->cxsr);
1517 drm_WARN_ON(&dev_priv->drm,
1518 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1519 intermediate->fbc_en && intermediate->hpll_en);
1523 * If our intermediate WM are identical to the final WM, then we can
1524 * omit the post-vblank programming; only update if it's different.
1526 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1527 new_crtc_state->wm.need_postvbl_update = true;
1532 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1533 struct g4x_wm_values *wm)
1535 struct intel_crtc *crtc;
1536 int num_active_pipes = 0;
1542 for_each_intel_crtc(&dev_priv->drm, crtc) {
1543 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1548 if (!wm_state->cxsr)
1550 if (!wm_state->hpll_en)
1551 wm->hpll_en = false;
1552 if (!wm_state->fbc_en)
1558 if (num_active_pipes != 1) {
1560 wm->hpll_en = false;
1564 for_each_intel_crtc(&dev_priv->drm, crtc) {
1565 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1566 enum pipe pipe = crtc->pipe;
1568 wm->pipe[pipe] = wm_state->wm;
1569 if (crtc->active && wm->cxsr)
1570 wm->sr = wm_state->sr;
1571 if (crtc->active && wm->hpll_en)
1572 wm->hpll = wm_state->hpll;
1576 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1578 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1579 struct g4x_wm_values new_wm = {};
1581 g4x_merge_wm(dev_priv, &new_wm);
1583 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1586 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1587 _intel_set_memory_cxsr(dev_priv, false);
1589 g4x_write_wm_values(dev_priv, &new_wm);
1591 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1592 _intel_set_memory_cxsr(dev_priv, true);
1597 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1598 struct intel_crtc *crtc)
1600 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1601 const struct intel_crtc_state *crtc_state =
1602 intel_atomic_get_new_crtc_state(state, crtc);
1604 mutex_lock(&dev_priv->wm.wm_mutex);
1605 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1606 g4x_program_watermarks(dev_priv);
1607 mutex_unlock(&dev_priv->wm.wm_mutex);
1610 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1611 struct intel_crtc *crtc)
1613 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1614 const struct intel_crtc_state *crtc_state =
1615 intel_atomic_get_new_crtc_state(state, crtc);
1617 if (!crtc_state->wm.need_postvbl_update)
1620 mutex_lock(&dev_priv->wm.wm_mutex);
1621 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1622 g4x_program_watermarks(dev_priv);
1623 mutex_unlock(&dev_priv->wm.wm_mutex);
1626 /* latency must be in 0.1us units. */
1627 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1628 unsigned int htotal,
1631 unsigned int latency)
1635 ret = intel_wm_method2(pixel_rate, htotal,
1636 width, cpp, latency);
1637 ret = DIV_ROUND_UP(ret, 64);
1642 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1644 /* all latencies in usec */
1645 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1647 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1649 if (IS_CHERRYVIEW(dev_priv)) {
1650 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1651 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1653 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1657 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1658 const struct intel_plane_state *plane_state,
1661 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1662 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1663 const struct drm_display_mode *adjusted_mode =
1664 &crtc_state->hw.adjusted_mode;
1665 unsigned int clock, htotal, cpp, width, wm;
1667 if (dev_priv->wm.pri_latency[level] == 0)
1670 if (!intel_wm_plane_visible(crtc_state, plane_state))
1673 cpp = plane_state->hw.fb->format->cpp[0];
1674 clock = adjusted_mode->crtc_clock;
1675 htotal = adjusted_mode->crtc_htotal;
1676 width = crtc_state->pipe_src_w;
1678 if (plane->id == PLANE_CURSOR) {
1680 * FIXME the formula gives values that are
1681 * too big for the cursor FIFO, and hence we
1682 * would never be able to use cursors. For
1683 * now just hardcode the watermark.
1687 wm = vlv_wm_method2(clock, htotal, width, cpp,
1688 dev_priv->wm.pri_latency[level] * 10);
1691 return min_t(unsigned int, wm, USHRT_MAX);
1694 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1696 return (active_planes & (BIT(PLANE_SPRITE0) |
1697 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1700 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1702 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1703 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1704 const struct g4x_pipe_wm *raw =
1705 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1706 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1707 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1708 int num_active_planes = hweight8(active_planes);
1709 const int fifo_size = 511;
1710 int fifo_extra, fifo_left = fifo_size;
1711 int sprite0_fifo_extra = 0;
1712 unsigned int total_rate;
1713 enum plane_id plane_id;
1716 * When enabling sprite0 after sprite1 has already been enabled
1717 * we tend to get an underrun unless sprite0 already has some
1718 * FIFO space allcoated. Hence we always allocate at least one
1719 * cacheline for sprite0 whenever sprite1 is enabled.
1721 * All other plane enable sequences appear immune to this problem.
1723 if (vlv_need_sprite0_fifo_workaround(active_planes))
1724 sprite0_fifo_extra = 1;
1726 total_rate = raw->plane[PLANE_PRIMARY] +
1727 raw->plane[PLANE_SPRITE0] +
1728 raw->plane[PLANE_SPRITE1] +
1731 if (total_rate > fifo_size)
1734 if (total_rate == 0)
1737 for_each_plane_id_on_crtc(crtc, plane_id) {
1740 if ((active_planes & BIT(plane_id)) == 0) {
1741 fifo_state->plane[plane_id] = 0;
1745 rate = raw->plane[plane_id];
1746 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1747 fifo_left -= fifo_state->plane[plane_id];
1750 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1751 fifo_left -= sprite0_fifo_extra;
1753 fifo_state->plane[PLANE_CURSOR] = 63;
1755 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1757 /* spread the remainder evenly */
1758 for_each_plane_id_on_crtc(crtc, plane_id) {
1764 if ((active_planes & BIT(plane_id)) == 0)
1767 plane_extra = min(fifo_extra, fifo_left);
1768 fifo_state->plane[plane_id] += plane_extra;
1769 fifo_left -= plane_extra;
1772 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1774 /* give it all to the first plane if none are active */
1775 if (active_planes == 0) {
1776 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1777 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1783 /* mark all levels starting from 'level' as invalid */
1784 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1785 struct vlv_wm_state *wm_state, int level)
1787 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1789 for (; level < intel_wm_num_levels(dev_priv); level++) {
1790 enum plane_id plane_id;
1792 for_each_plane_id_on_crtc(crtc, plane_id)
1793 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1795 wm_state->sr[level].cursor = USHRT_MAX;
1796 wm_state->sr[level].plane = USHRT_MAX;
1800 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1805 return fifo_size - wm;
1809 * Starting from 'level' set all higher
1810 * levels to 'value' in the "raw" watermarks.
1812 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1813 int level, enum plane_id plane_id, u16 value)
1815 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1816 int num_levels = intel_wm_num_levels(dev_priv);
1819 for (; level < num_levels; level++) {
1820 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1822 dirty |= raw->plane[plane_id] != value;
1823 raw->plane[plane_id] = value;
1829 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1830 const struct intel_plane_state *plane_state)
1832 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1833 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1834 enum plane_id plane_id = plane->id;
1835 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1839 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1840 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1844 for (level = 0; level < num_levels; level++) {
1845 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1846 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1847 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1852 dirty |= raw->plane[plane_id] != wm;
1853 raw->plane[plane_id] = wm;
1856 /* mark all higher levels as invalid */
1857 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1861 drm_dbg_kms(&dev_priv->drm,
1862 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1864 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1865 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1866 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1871 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1872 enum plane_id plane_id, int level)
1874 const struct g4x_pipe_wm *raw =
1875 &crtc_state->wm.vlv.raw[level];
1876 const struct vlv_fifo_state *fifo_state =
1877 &crtc_state->wm.vlv.fifo_state;
1879 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1882 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1884 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1885 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1886 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1887 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1890 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1892 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1893 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1894 struct intel_atomic_state *state =
1895 to_intel_atomic_state(crtc_state->uapi.state);
1896 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1897 const struct vlv_fifo_state *fifo_state =
1898 &crtc_state->wm.vlv.fifo_state;
1899 int num_active_planes = hweight8(crtc_state->active_planes &
1900 ~BIT(PLANE_CURSOR));
1901 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1902 const struct intel_plane_state *old_plane_state;
1903 const struct intel_plane_state *new_plane_state;
1904 struct intel_plane *plane;
1905 enum plane_id plane_id;
1907 unsigned int dirty = 0;
1909 for_each_oldnew_intel_plane_in_state(state, plane,
1911 new_plane_state, i) {
1912 if (new_plane_state->hw.crtc != &crtc->base &&
1913 old_plane_state->hw.crtc != &crtc->base)
1916 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1917 dirty |= BIT(plane->id);
1921 * DSPARB registers may have been reset due to the
1922 * power well being turned off. Make sure we restore
1923 * them to a consistent state even if no primary/sprite
1924 * planes are initially active.
1927 crtc_state->fifo_changed = true;
1932 /* cursor changes don't warrant a FIFO recompute */
1933 if (dirty & ~BIT(PLANE_CURSOR)) {
1934 const struct intel_crtc_state *old_crtc_state =
1935 intel_atomic_get_old_crtc_state(state, crtc);
1936 const struct vlv_fifo_state *old_fifo_state =
1937 &old_crtc_state->wm.vlv.fifo_state;
1939 ret = vlv_compute_fifo(crtc_state);
1943 if (needs_modeset ||
1944 memcmp(old_fifo_state, fifo_state,
1945 sizeof(*fifo_state)) != 0)
1946 crtc_state->fifo_changed = true;
1949 /* initially allow all levels */
1950 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1952 * Note that enabling cxsr with no primary/sprite planes
1953 * enabled can wedge the pipe. Hence we only allow cxsr
1954 * with exactly one enabled primary/sprite plane.
1956 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1958 for (level = 0; level < wm_state->num_levels; level++) {
1959 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1960 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1962 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1965 for_each_plane_id_on_crtc(crtc, plane_id) {
1966 wm_state->wm[level].plane[plane_id] =
1967 vlv_invert_wm_value(raw->plane[plane_id],
1968 fifo_state->plane[plane_id]);
1971 wm_state->sr[level].plane =
1972 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1973 raw->plane[PLANE_SPRITE0],
1974 raw->plane[PLANE_SPRITE1]),
1977 wm_state->sr[level].cursor =
1978 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1985 /* limit to only levels we can actually handle */
1986 wm_state->num_levels = level;
1988 /* invalidate the higher levels */
1989 vlv_invalidate_wms(crtc, wm_state, level);
1994 #define VLV_FIFO(plane, value) \
1995 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1997 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1998 struct intel_crtc *crtc)
2000 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2001 struct intel_uncore *uncore = &dev_priv->uncore;
2002 const struct intel_crtc_state *crtc_state =
2003 intel_atomic_get_new_crtc_state(state, crtc);
2004 const struct vlv_fifo_state *fifo_state =
2005 &crtc_state->wm.vlv.fifo_state;
2006 int sprite0_start, sprite1_start, fifo_size;
2007 u32 dsparb, dsparb2, dsparb3;
2009 if (!crtc_state->fifo_changed)
2012 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2013 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2014 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2016 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2017 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2019 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2022 * uncore.lock serves a double purpose here. It allows us to
2023 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2024 * it protects the DSPARB registers from getting clobbered by
2025 * parallel updates from multiple pipes.
2027 * intel_pipe_update_start() has already disabled interrupts
2028 * for us, so a plain spin_lock() is sufficient here.
2030 spin_lock(&uncore->lock);
2032 switch (crtc->pipe) {
2034 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2035 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2037 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2038 VLV_FIFO(SPRITEB, 0xff));
2039 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2040 VLV_FIFO(SPRITEB, sprite1_start));
2042 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2043 VLV_FIFO(SPRITEB_HI, 0x1));
2044 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2045 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2047 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2048 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2051 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2052 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2054 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2055 VLV_FIFO(SPRITED, 0xff));
2056 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2057 VLV_FIFO(SPRITED, sprite1_start));
2059 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2060 VLV_FIFO(SPRITED_HI, 0xff));
2061 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2062 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2064 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2065 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2068 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2069 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2071 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2072 VLV_FIFO(SPRITEF, 0xff));
2073 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2074 VLV_FIFO(SPRITEF, sprite1_start));
2076 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2077 VLV_FIFO(SPRITEF_HI, 0xff));
2078 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2079 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2081 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2082 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2088 intel_uncore_posting_read_fw(uncore, DSPARB);
2090 spin_unlock(&uncore->lock);
2095 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2097 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2098 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2099 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2100 struct intel_atomic_state *intel_state =
2101 to_intel_atomic_state(new_crtc_state->uapi.state);
2102 const struct intel_crtc_state *old_crtc_state =
2103 intel_atomic_get_old_crtc_state(intel_state, crtc);
2104 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2107 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2108 *intermediate = *optimal;
2110 intermediate->cxsr = false;
2114 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2115 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2116 !new_crtc_state->disable_cxsr;
2118 for (level = 0; level < intermediate->num_levels; level++) {
2119 enum plane_id plane_id;
2121 for_each_plane_id_on_crtc(crtc, plane_id) {
2122 intermediate->wm[level].plane[plane_id] =
2123 min(optimal->wm[level].plane[plane_id],
2124 active->wm[level].plane[plane_id]);
2127 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2128 active->sr[level].plane);
2129 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2130 active->sr[level].cursor);
2133 vlv_invalidate_wms(crtc, intermediate, level);
2137 * If our intermediate WM are identical to the final WM, then we can
2138 * omit the post-vblank programming; only update if it's different.
2140 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2141 new_crtc_state->wm.need_postvbl_update = true;
2146 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2147 struct vlv_wm_values *wm)
2149 struct intel_crtc *crtc;
2150 int num_active_pipes = 0;
2152 wm->level = dev_priv->wm.max_level;
2155 for_each_intel_crtc(&dev_priv->drm, crtc) {
2156 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2161 if (!wm_state->cxsr)
2165 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2168 if (num_active_pipes != 1)
2171 if (num_active_pipes > 1)
2172 wm->level = VLV_WM_LEVEL_PM2;
2174 for_each_intel_crtc(&dev_priv->drm, crtc) {
2175 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2176 enum pipe pipe = crtc->pipe;
2178 wm->pipe[pipe] = wm_state->wm[wm->level];
2179 if (crtc->active && wm->cxsr)
2180 wm->sr = wm_state->sr[wm->level];
2182 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2183 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2184 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2185 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2189 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2191 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2192 struct vlv_wm_values new_wm = {};
2194 vlv_merge_wm(dev_priv, &new_wm);
2196 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2199 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2200 chv_set_memory_dvfs(dev_priv, false);
2202 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2203 chv_set_memory_pm5(dev_priv, false);
2205 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2206 _intel_set_memory_cxsr(dev_priv, false);
2208 vlv_write_wm_values(dev_priv, &new_wm);
2210 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2211 _intel_set_memory_cxsr(dev_priv, true);
2213 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2214 chv_set_memory_pm5(dev_priv, true);
2216 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2217 chv_set_memory_dvfs(dev_priv, true);
2222 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2223 struct intel_crtc *crtc)
2225 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2226 const struct intel_crtc_state *crtc_state =
2227 intel_atomic_get_new_crtc_state(state, crtc);
2229 mutex_lock(&dev_priv->wm.wm_mutex);
2230 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2231 vlv_program_watermarks(dev_priv);
2232 mutex_unlock(&dev_priv->wm.wm_mutex);
2235 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2236 struct intel_crtc *crtc)
2238 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2239 const struct intel_crtc_state *crtc_state =
2240 intel_atomic_get_new_crtc_state(state, crtc);
2242 if (!crtc_state->wm.need_postvbl_update)
2245 mutex_lock(&dev_priv->wm.wm_mutex);
2246 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2247 vlv_program_watermarks(dev_priv);
2248 mutex_unlock(&dev_priv->wm.wm_mutex);
2251 static void i965_update_wm(struct intel_crtc *unused_crtc)
2253 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2254 struct intel_crtc *crtc;
2259 /* Calc sr entries for one plane configs */
2260 crtc = single_enabled_crtc(dev_priv);
2262 /* self-refresh has much higher latency */
2263 static const int sr_latency_ns = 12000;
2264 const struct drm_display_mode *adjusted_mode =
2265 &crtc->config->hw.adjusted_mode;
2266 const struct drm_framebuffer *fb =
2267 crtc->base.primary->state->fb;
2268 int clock = adjusted_mode->crtc_clock;
2269 int htotal = adjusted_mode->crtc_htotal;
2270 int hdisplay = crtc->config->pipe_src_w;
2271 int cpp = fb->format->cpp[0];
2274 entries = intel_wm_method2(clock, htotal,
2275 hdisplay, cpp, sr_latency_ns / 100);
2276 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2277 srwm = I965_FIFO_SIZE - entries;
2281 drm_dbg_kms(&dev_priv->drm,
2282 "self-refresh entries: %d, wm: %d\n",
2285 entries = intel_wm_method2(clock, htotal,
2286 crtc->base.cursor->state->crtc_w, 4,
2287 sr_latency_ns / 100);
2288 entries = DIV_ROUND_UP(entries,
2289 i965_cursor_wm_info.cacheline_size) +
2290 i965_cursor_wm_info.guard_size;
2292 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2293 if (cursor_sr > i965_cursor_wm_info.max_wm)
2294 cursor_sr = i965_cursor_wm_info.max_wm;
2296 drm_dbg_kms(&dev_priv->drm,
2297 "self-refresh watermark: display plane %d "
2298 "cursor %d\n", srwm, cursor_sr);
2300 cxsr_enabled = true;
2302 cxsr_enabled = false;
2303 /* Turn off self refresh if both pipes are enabled */
2304 intel_set_memory_cxsr(dev_priv, false);
2307 drm_dbg_kms(&dev_priv->drm,
2308 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2311 /* 965 has limitations... */
2312 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2316 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2317 FW_WM(8, PLANEC_OLD));
2318 /* update cursor SR watermark */
2319 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2322 intel_set_memory_cxsr(dev_priv, true);
2327 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2329 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2330 const struct intel_watermark_params *wm_info;
2335 int planea_wm, planeb_wm;
2336 struct intel_crtc *crtc, *enabled = NULL;
2338 if (IS_I945GM(dev_priv))
2339 wm_info = &i945_wm_info;
2340 else if (!IS_GEN(dev_priv, 2))
2341 wm_info = &i915_wm_info;
2343 wm_info = &i830_a_wm_info;
2345 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2346 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2347 if (intel_crtc_active(crtc)) {
2348 const struct drm_display_mode *adjusted_mode =
2349 &crtc->config->hw.adjusted_mode;
2350 const struct drm_framebuffer *fb =
2351 crtc->base.primary->state->fb;
2354 if (IS_GEN(dev_priv, 2))
2357 cpp = fb->format->cpp[0];
2359 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2360 wm_info, fifo_size, cpp,
2361 pessimal_latency_ns);
2364 planea_wm = fifo_size - wm_info->guard_size;
2365 if (planea_wm > (long)wm_info->max_wm)
2366 planea_wm = wm_info->max_wm;
2369 if (IS_GEN(dev_priv, 2))
2370 wm_info = &i830_bc_wm_info;
2372 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2373 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2374 if (intel_crtc_active(crtc)) {
2375 const struct drm_display_mode *adjusted_mode =
2376 &crtc->config->hw.adjusted_mode;
2377 const struct drm_framebuffer *fb =
2378 crtc->base.primary->state->fb;
2381 if (IS_GEN(dev_priv, 2))
2384 cpp = fb->format->cpp[0];
2386 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2387 wm_info, fifo_size, cpp,
2388 pessimal_latency_ns);
2389 if (enabled == NULL)
2394 planeb_wm = fifo_size - wm_info->guard_size;
2395 if (planeb_wm > (long)wm_info->max_wm)
2396 planeb_wm = wm_info->max_wm;
2399 drm_dbg_kms(&dev_priv->drm,
2400 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2402 if (IS_I915GM(dev_priv) && enabled) {
2403 struct drm_i915_gem_object *obj;
2405 obj = intel_fb_obj(enabled->base.primary->state->fb);
2407 /* self-refresh seems busted with untiled */
2408 if (!i915_gem_object_is_tiled(obj))
2413 * Overlay gets an aggressive default since video jitter is bad.
2417 /* Play safe and disable self-refresh before adjusting watermarks. */
2418 intel_set_memory_cxsr(dev_priv, false);
2420 /* Calc sr entries for one plane configs */
2421 if (HAS_FW_BLC(dev_priv) && enabled) {
2422 /* self-refresh has much higher latency */
2423 static const int sr_latency_ns = 6000;
2424 const struct drm_display_mode *adjusted_mode =
2425 &enabled->config->hw.adjusted_mode;
2426 const struct drm_framebuffer *fb =
2427 enabled->base.primary->state->fb;
2428 int clock = adjusted_mode->crtc_clock;
2429 int htotal = adjusted_mode->crtc_htotal;
2430 int hdisplay = enabled->config->pipe_src_w;
2434 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2437 cpp = fb->format->cpp[0];
2439 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2440 sr_latency_ns / 100);
2441 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2442 drm_dbg_kms(&dev_priv->drm,
2443 "self-refresh entries: %d\n", entries);
2444 srwm = wm_info->fifo_size - entries;
2448 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2449 I915_WRITE(FW_BLC_SELF,
2450 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2452 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2455 drm_dbg_kms(&dev_priv->drm,
2456 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2457 planea_wm, planeb_wm, cwm, srwm);
2459 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2460 fwater_hi = (cwm & 0x1f);
2462 /* Set request length to 8 cachelines per fetch */
2463 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2464 fwater_hi = fwater_hi | (1 << 8);
2466 I915_WRITE(FW_BLC, fwater_lo);
2467 I915_WRITE(FW_BLC2, fwater_hi);
2470 intel_set_memory_cxsr(dev_priv, true);
2473 static void i845_update_wm(struct intel_crtc *unused_crtc)
2475 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2476 struct intel_crtc *crtc;
2477 const struct drm_display_mode *adjusted_mode;
2481 crtc = single_enabled_crtc(dev_priv);
2485 adjusted_mode = &crtc->config->hw.adjusted_mode;
2486 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2488 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2489 4, pessimal_latency_ns);
2490 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2491 fwater_lo |= (3<<8) | planea_wm;
2493 drm_dbg_kms(&dev_priv->drm,
2494 "Setting FIFO watermarks - A: %d\n", planea_wm);
2496 I915_WRITE(FW_BLC, fwater_lo);
2499 /* latency must be in 0.1us units. */
2500 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2502 unsigned int latency)
2506 ret = intel_wm_method1(pixel_rate, cpp, latency);
2507 ret = DIV_ROUND_UP(ret, 64) + 2;
2512 /* latency must be in 0.1us units. */
2513 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2514 unsigned int htotal,
2517 unsigned int latency)
2521 ret = intel_wm_method2(pixel_rate, htotal,
2522 width, cpp, latency);
2523 ret = DIV_ROUND_UP(ret, 64) + 2;
2528 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2531 * Neither of these should be possible since this function shouldn't be
2532 * called if the CRTC is off or the plane is invisible. But let's be
2533 * extra paranoid to avoid a potential divide-by-zero if we screw up
2534 * elsewhere in the driver.
2538 if (WARN_ON(!horiz_pixels))
2541 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2544 struct ilk_wm_maximums {
2552 * For both WM_PIPE and WM_LP.
2553 * mem_value must be in 0.1us units.
2555 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2556 const struct intel_plane_state *plane_state,
2557 u32 mem_value, bool is_lp)
2559 u32 method1, method2;
2565 if (!intel_wm_plane_visible(crtc_state, plane_state))
2568 cpp = plane_state->hw.fb->format->cpp[0];
2570 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2575 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2576 crtc_state->hw.adjusted_mode.crtc_htotal,
2577 drm_rect_width(&plane_state->uapi.dst),
2580 return min(method1, method2);
2584 * For both WM_PIPE and WM_LP.
2585 * mem_value must be in 0.1us units.
2587 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2588 const struct intel_plane_state *plane_state,
2591 u32 method1, method2;
2597 if (!intel_wm_plane_visible(crtc_state, plane_state))
2600 cpp = plane_state->hw.fb->format->cpp[0];
2602 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2603 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2604 crtc_state->hw.adjusted_mode.crtc_htotal,
2605 drm_rect_width(&plane_state->uapi.dst),
2607 return min(method1, method2);
2611 * For both WM_PIPE and WM_LP.
2612 * mem_value must be in 0.1us units.
2614 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2615 const struct intel_plane_state *plane_state,
2623 if (!intel_wm_plane_visible(crtc_state, plane_state))
2626 cpp = plane_state->hw.fb->format->cpp[0];
2628 return ilk_wm_method2(crtc_state->pixel_rate,
2629 crtc_state->hw.adjusted_mode.crtc_htotal,
2630 drm_rect_width(&plane_state->uapi.dst),
2634 /* Only for WM_LP. */
2635 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2636 const struct intel_plane_state *plane_state,
2641 if (!intel_wm_plane_visible(crtc_state, plane_state))
2644 cpp = plane_state->hw.fb->format->cpp[0];
2646 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2651 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2653 if (INTEL_GEN(dev_priv) >= 8)
2655 else if (INTEL_GEN(dev_priv) >= 7)
2662 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2663 int level, bool is_sprite)
2665 if (INTEL_GEN(dev_priv) >= 8)
2666 /* BDW primary/sprite plane watermarks */
2667 return level == 0 ? 255 : 2047;
2668 else if (INTEL_GEN(dev_priv) >= 7)
2669 /* IVB/HSW primary/sprite plane watermarks */
2670 return level == 0 ? 127 : 1023;
2671 else if (!is_sprite)
2672 /* ILK/SNB primary plane watermarks */
2673 return level == 0 ? 127 : 511;
2675 /* ILK/SNB sprite plane watermarks */
2676 return level == 0 ? 63 : 255;
2680 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2682 if (INTEL_GEN(dev_priv) >= 7)
2683 return level == 0 ? 63 : 255;
2685 return level == 0 ? 31 : 63;
2688 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2690 if (INTEL_GEN(dev_priv) >= 8)
2696 /* Calculate the maximum primary/sprite plane watermark */
2697 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2699 const struct intel_wm_config *config,
2700 enum intel_ddb_partitioning ddb_partitioning,
2703 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2705 /* if sprites aren't enabled, sprites get nothing */
2706 if (is_sprite && !config->sprites_enabled)
2709 /* HSW allows LP1+ watermarks even with multiple pipes */
2710 if (level == 0 || config->num_pipes_active > 1) {
2711 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2714 * For some reason the non self refresh
2715 * FIFO size is only half of the self
2716 * refresh FIFO size on ILK/SNB.
2718 if (INTEL_GEN(dev_priv) <= 6)
2722 if (config->sprites_enabled) {
2723 /* level 0 is always calculated with 1:1 split */
2724 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2733 /* clamp to max that the registers can hold */
2734 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2737 /* Calculate the maximum cursor plane watermark */
2738 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2740 const struct intel_wm_config *config)
2742 /* HSW LP1+ watermarks w/ multiple pipes */
2743 if (level > 0 && config->num_pipes_active > 1)
2746 /* otherwise just report max that registers can hold */
2747 return ilk_cursor_wm_reg_max(dev_priv, level);
2750 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2752 const struct intel_wm_config *config,
2753 enum intel_ddb_partitioning ddb_partitioning,
2754 struct ilk_wm_maximums *max)
2756 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2757 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2758 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2759 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2762 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2764 struct ilk_wm_maximums *max)
2766 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2767 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2768 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2769 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2772 static bool ilk_validate_wm_level(int level,
2773 const struct ilk_wm_maximums *max,
2774 struct intel_wm_level *result)
2778 /* already determined to be invalid? */
2779 if (!result->enable)
2782 result->enable = result->pri_val <= max->pri &&
2783 result->spr_val <= max->spr &&
2784 result->cur_val <= max->cur;
2786 ret = result->enable;
2789 * HACK until we can pre-compute everything,
2790 * and thus fail gracefully if LP0 watermarks
2793 if (level == 0 && !result->enable) {
2794 if (result->pri_val > max->pri)
2795 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2796 level, result->pri_val, max->pri);
2797 if (result->spr_val > max->spr)
2798 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2799 level, result->spr_val, max->spr);
2800 if (result->cur_val > max->cur)
2801 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2802 level, result->cur_val, max->cur);
2804 result->pri_val = min_t(u32, result->pri_val, max->pri);
2805 result->spr_val = min_t(u32, result->spr_val, max->spr);
2806 result->cur_val = min_t(u32, result->cur_val, max->cur);
2807 result->enable = true;
2813 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2814 const struct intel_crtc *crtc,
2816 struct intel_crtc_state *crtc_state,
2817 const struct intel_plane_state *pristate,
2818 const struct intel_plane_state *sprstate,
2819 const struct intel_plane_state *curstate,
2820 struct intel_wm_level *result)
2822 u16 pri_latency = dev_priv->wm.pri_latency[level];
2823 u16 spr_latency = dev_priv->wm.spr_latency[level];
2824 u16 cur_latency = dev_priv->wm.cur_latency[level];
2826 /* WM1+ latency values stored in 0.5us units */
2834 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2835 pri_latency, level);
2836 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2840 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2843 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2845 result->enable = true;
2848 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2851 struct intel_uncore *uncore = &dev_priv->uncore;
2853 if (INTEL_GEN(dev_priv) >= 9) {
2856 int level, max_level = ilk_wm_max_level(dev_priv);
2858 /* read the first set of memory latencies[0:3] */
2859 val = 0; /* data0 to be programmed to 0 for first set */
2860 ret = sandybridge_pcode_read(dev_priv,
2861 GEN9_PCODE_READ_MEM_LATENCY,
2865 drm_err(&dev_priv->drm,
2866 "SKL Mailbox read error = %d\n", ret);
2870 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2871 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2872 GEN9_MEM_LATENCY_LEVEL_MASK;
2873 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2874 GEN9_MEM_LATENCY_LEVEL_MASK;
2875 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2876 GEN9_MEM_LATENCY_LEVEL_MASK;
2878 /* read the second set of memory latencies[4:7] */
2879 val = 1; /* data0 to be programmed to 1 for second set */
2880 ret = sandybridge_pcode_read(dev_priv,
2881 GEN9_PCODE_READ_MEM_LATENCY,
2884 drm_err(&dev_priv->drm,
2885 "SKL Mailbox read error = %d\n", ret);
2889 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2890 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2891 GEN9_MEM_LATENCY_LEVEL_MASK;
2892 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2893 GEN9_MEM_LATENCY_LEVEL_MASK;
2894 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2895 GEN9_MEM_LATENCY_LEVEL_MASK;
2898 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2899 * need to be disabled. We make sure to sanitize the values out
2900 * of the punit to satisfy this requirement.
2902 for (level = 1; level <= max_level; level++) {
2903 if (wm[level] == 0) {
2904 for (i = level + 1; i <= max_level; i++)
2911 * WaWmMemoryReadLatency:skl+,glk
2913 * punit doesn't take into account the read latency so we need
2914 * to add 2us to the various latency levels we retrieve from the
2915 * punit when level 0 response data us 0us.
2919 for (level = 1; level <= max_level; level++) {
2927 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2928 * If we could not get dimm info enable this WA to prevent from
2929 * any underrun. If not able to get Dimm info assume 16GB dimm
2930 * to avoid any underrun.
2932 if (dev_priv->dram_info.is_16gb_dimm)
2935 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2936 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2938 wm[0] = (sskpd >> 56) & 0xFF;
2940 wm[0] = sskpd & 0xF;
2941 wm[1] = (sskpd >> 4) & 0xFF;
2942 wm[2] = (sskpd >> 12) & 0xFF;
2943 wm[3] = (sskpd >> 20) & 0x1FF;
2944 wm[4] = (sskpd >> 32) & 0x1FF;
2945 } else if (INTEL_GEN(dev_priv) >= 6) {
2946 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2948 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2949 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2950 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2951 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2952 } else if (INTEL_GEN(dev_priv) >= 5) {
2953 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2955 /* ILK primary LP0 latency is 700 ns */
2957 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2958 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2960 MISSING_CASE(INTEL_DEVID(dev_priv));
2964 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2967 /* ILK sprite LP0 latency is 1300 ns */
2968 if (IS_GEN(dev_priv, 5))
2972 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2975 /* ILK cursor LP0 latency is 1300 ns */
2976 if (IS_GEN(dev_priv, 5))
2980 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2982 /* how many WM levels are we expecting */
2983 if (INTEL_GEN(dev_priv) >= 9)
2985 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2987 else if (INTEL_GEN(dev_priv) >= 6)
2993 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2997 int level, max_level = ilk_wm_max_level(dev_priv);
2999 for (level = 0; level <= max_level; level++) {
3000 unsigned int latency = wm[level];
3003 drm_dbg_kms(&dev_priv->drm,
3004 "%s WM%d latency not provided\n",
3010 * - latencies are in us on gen9.
3011 * - before then, WM1+ latency values are in 0.5us units
3013 if (INTEL_GEN(dev_priv) >= 9)
3018 drm_dbg_kms(&dev_priv->drm,
3019 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3020 wm[level], latency / 10, latency % 10);
3024 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3027 int level, max_level = ilk_wm_max_level(dev_priv);
3032 wm[0] = max(wm[0], min);
3033 for (level = 1; level <= max_level; level++)
3034 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3039 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3044 * The BIOS provided WM memory latency values are often
3045 * inadequate for high resolution displays. Adjust them.
3047 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3048 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3049 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3054 drm_dbg_kms(&dev_priv->drm,
3055 "WM latency values increased to avoid potential underruns\n");
3056 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3057 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3058 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3061 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3064 * On some SNB machines (Thinkpad X220 Tablet at least)
3065 * LP3 usage can cause vblank interrupts to be lost.
3066 * The DEIIR bit will go high but it looks like the CPU
3067 * never gets interrupted.
3069 * It's not clear whether other interrupt source could
3070 * be affected or if this is somehow limited to vblank
3071 * interrupts only. To play it safe we disable LP3
3072 * watermarks entirely.
3074 if (dev_priv->wm.pri_latency[3] == 0 &&
3075 dev_priv->wm.spr_latency[3] == 0 &&
3076 dev_priv->wm.cur_latency[3] == 0)
3079 dev_priv->wm.pri_latency[3] = 0;
3080 dev_priv->wm.spr_latency[3] = 0;
3081 dev_priv->wm.cur_latency[3] = 0;
3083 drm_dbg_kms(&dev_priv->drm,
3084 "LP3 watermarks disabled due to potential for lost interrupts\n");
3085 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3086 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3087 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3090 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3092 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3094 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3095 sizeof(dev_priv->wm.pri_latency));
3096 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3097 sizeof(dev_priv->wm.pri_latency));
3099 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3100 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3102 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3103 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3104 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3106 if (IS_GEN(dev_priv, 6)) {
3107 snb_wm_latency_quirk(dev_priv);
3108 snb_wm_lp3_irq_quirk(dev_priv);
3112 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3114 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3115 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3118 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3119 struct intel_pipe_wm *pipe_wm)
3121 /* LP0 watermark maximums depend on this pipe alone */
3122 const struct intel_wm_config config = {
3123 .num_pipes_active = 1,
3124 .sprites_enabled = pipe_wm->sprites_enabled,
3125 .sprites_scaled = pipe_wm->sprites_scaled,
3127 struct ilk_wm_maximums max;
3129 /* LP0 watermarks always use 1/2 DDB partitioning */
3130 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3132 /* At least LP0 must be valid */
3133 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3134 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3141 /* Compute new watermarks for the pipe */
3142 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3144 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3145 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3146 struct intel_pipe_wm *pipe_wm;
3147 struct intel_plane *plane;
3148 const struct intel_plane_state *plane_state;
3149 const struct intel_plane_state *pristate = NULL;
3150 const struct intel_plane_state *sprstate = NULL;
3151 const struct intel_plane_state *curstate = NULL;
3152 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3153 struct ilk_wm_maximums max;
3155 pipe_wm = &crtc_state->wm.ilk.optimal;
3157 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3158 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3159 pristate = plane_state;
3160 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3161 sprstate = plane_state;
3162 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3163 curstate = plane_state;
3166 pipe_wm->pipe_enabled = crtc_state->hw.active;
3168 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3169 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3170 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3171 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3174 usable_level = max_level;
3176 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3177 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3180 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3181 if (pipe_wm->sprites_scaled)
3184 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3185 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3186 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3188 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3191 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3193 for (level = 1; level <= usable_level; level++) {
3194 struct intel_wm_level *wm = &pipe_wm->wm[level];
3196 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3197 pristate, sprstate, curstate, wm);
3200 * Disable any watermark level that exceeds the
3201 * register maximums since such watermarks are
3204 if (!ilk_validate_wm_level(level, &max, wm)) {
3205 memset(wm, 0, sizeof(*wm));
3214 * Build a set of 'intermediate' watermark values that satisfy both the old
3215 * state and the new state. These can be programmed to the hardware
3218 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3220 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3221 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3222 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3223 struct intel_atomic_state *intel_state =
3224 to_intel_atomic_state(newstate->uapi.state);
3225 const struct intel_crtc_state *oldstate =
3226 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3227 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3228 int level, max_level = ilk_wm_max_level(dev_priv);
3231 * Start with the final, target watermarks, then combine with the
3232 * currently active watermarks to get values that are safe both before
3233 * and after the vblank.
3235 *a = newstate->wm.ilk.optimal;
3236 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3237 intel_state->skip_intermediate_wm)
3240 a->pipe_enabled |= b->pipe_enabled;
3241 a->sprites_enabled |= b->sprites_enabled;
3242 a->sprites_scaled |= b->sprites_scaled;
3244 for (level = 0; level <= max_level; level++) {
3245 struct intel_wm_level *a_wm = &a->wm[level];
3246 const struct intel_wm_level *b_wm = &b->wm[level];
3248 a_wm->enable &= b_wm->enable;
3249 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3250 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3251 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3252 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3256 * We need to make sure that these merged watermark values are
3257 * actually a valid configuration themselves. If they're not,
3258 * there's no safe way to transition from the old state to
3259 * the new state, so we need to fail the atomic transaction.
3261 if (!ilk_validate_pipe_wm(dev_priv, a))
3265 * If our intermediate WM are identical to the final WM, then we can
3266 * omit the post-vblank programming; only update if it's different.
3268 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3269 newstate->wm.need_postvbl_update = true;
3275 * Merge the watermarks from all active pipes for a specific level.
3277 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3279 struct intel_wm_level *ret_wm)
3281 const struct intel_crtc *intel_crtc;
3283 ret_wm->enable = true;
3285 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3286 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3287 const struct intel_wm_level *wm = &active->wm[level];
3289 if (!active->pipe_enabled)
3293 * The watermark values may have been used in the past,
3294 * so we must maintain them in the registers for some
3295 * time even if the level is now disabled.
3298 ret_wm->enable = false;
3300 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3301 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3302 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3303 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3308 * Merge all low power watermarks for all active pipes.
3310 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3311 const struct intel_wm_config *config,
3312 const struct ilk_wm_maximums *max,
3313 struct intel_pipe_wm *merged)
3315 int level, max_level = ilk_wm_max_level(dev_priv);
3316 int last_enabled_level = max_level;
3318 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3319 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3320 config->num_pipes_active > 1)
3321 last_enabled_level = 0;
3323 /* ILK: FBC WM must be disabled always */
3324 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3326 /* merge each WM1+ level */
3327 for (level = 1; level <= max_level; level++) {
3328 struct intel_wm_level *wm = &merged->wm[level];
3330 ilk_merge_wm_level(dev_priv, level, wm);
3332 if (level > last_enabled_level)
3334 else if (!ilk_validate_wm_level(level, max, wm))
3335 /* make sure all following levels get disabled */
3336 last_enabled_level = level - 1;
3339 * The spec says it is preferred to disable
3340 * FBC WMs instead of disabling a WM level.
3342 if (wm->fbc_val > max->fbc) {
3344 merged->fbc_wm_enabled = false;
3349 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3351 * FIXME this is racy. FBC might get enabled later.
3352 * What we should check here is whether FBC can be
3353 * enabled sometime later.
3355 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3356 intel_fbc_is_active(dev_priv)) {
3357 for (level = 2; level <= max_level; level++) {
3358 struct intel_wm_level *wm = &merged->wm[level];
3365 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3367 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3368 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3371 /* The value we need to program into the WM_LPx latency field */
3372 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3375 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3378 return dev_priv->wm.pri_latency[level];
3381 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3382 const struct intel_pipe_wm *merged,
3383 enum intel_ddb_partitioning partitioning,
3384 struct ilk_wm_values *results)
3386 struct intel_crtc *intel_crtc;
3389 results->enable_fbc_wm = merged->fbc_wm_enabled;
3390 results->partitioning = partitioning;
3392 /* LP1+ register values */
3393 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3394 const struct intel_wm_level *r;
3396 level = ilk_wm_lp_to_level(wm_lp, merged);
3398 r = &merged->wm[level];
3401 * Maintain the watermark values even if the level is
3402 * disabled. Doing otherwise could cause underruns.
3404 results->wm_lp[wm_lp - 1] =
3405 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3406 (r->pri_val << WM1_LP_SR_SHIFT) |
3410 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3412 if (INTEL_GEN(dev_priv) >= 8)
3413 results->wm_lp[wm_lp - 1] |=
3414 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3416 results->wm_lp[wm_lp - 1] |=
3417 r->fbc_val << WM1_LP_FBC_SHIFT;
3420 * Always set WM1S_LP_EN when spr_val != 0, even if the
3421 * level is disabled. Doing otherwise could cause underruns.
3423 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3424 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3425 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3427 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3430 /* LP0 register values */
3431 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3432 enum pipe pipe = intel_crtc->pipe;
3433 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3434 const struct intel_wm_level *r = &pipe_wm->wm[0];
3436 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3439 results->wm_pipe[pipe] =
3440 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3441 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3446 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3447 * case both are at the same level. Prefer r1 in case they're the same. */
3448 static struct intel_pipe_wm *
3449 ilk_find_best_result(struct drm_i915_private *dev_priv,
3450 struct intel_pipe_wm *r1,
3451 struct intel_pipe_wm *r2)
3453 int level, max_level = ilk_wm_max_level(dev_priv);
3454 int level1 = 0, level2 = 0;
3456 for (level = 1; level <= max_level; level++) {
3457 if (r1->wm[level].enable)
3459 if (r2->wm[level].enable)
3463 if (level1 == level2) {
3464 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3468 } else if (level1 > level2) {
3475 /* dirty bits used to track which watermarks need changes */
3476 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3477 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3478 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3479 #define WM_DIRTY_FBC (1 << 24)
3480 #define WM_DIRTY_DDB (1 << 25)
3482 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3483 const struct ilk_wm_values *old,
3484 const struct ilk_wm_values *new)
3486 unsigned int dirty = 0;
3490 for_each_pipe(dev_priv, pipe) {
3491 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3492 dirty |= WM_DIRTY_PIPE(pipe);
3493 /* Must disable LP1+ watermarks too */
3494 dirty |= WM_DIRTY_LP_ALL;
3498 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3499 dirty |= WM_DIRTY_FBC;
3500 /* Must disable LP1+ watermarks too */
3501 dirty |= WM_DIRTY_LP_ALL;
3504 if (old->partitioning != new->partitioning) {
3505 dirty |= WM_DIRTY_DDB;
3506 /* Must disable LP1+ watermarks too */
3507 dirty |= WM_DIRTY_LP_ALL;
3510 /* LP1+ watermarks already deemed dirty, no need to continue */
3511 if (dirty & WM_DIRTY_LP_ALL)
3514 /* Find the lowest numbered LP1+ watermark in need of an update... */
3515 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3516 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3517 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3521 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3522 for (; wm_lp <= 3; wm_lp++)
3523 dirty |= WM_DIRTY_LP(wm_lp);
3528 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3531 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3532 bool changed = false;
3534 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3535 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3536 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3539 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3540 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3541 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3544 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3545 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3546 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3551 * Don't touch WM1S_LP_EN here.
3552 * Doing so could cause underruns.
3559 * The spec says we shouldn't write when we don't need, because every write
3560 * causes WMs to be re-evaluated, expending some power.
3562 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3563 struct ilk_wm_values *results)
3565 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3569 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3573 _ilk_disable_lp_wm(dev_priv, dirty);
3575 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3576 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3577 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3578 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3579 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3580 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3582 if (dirty & WM_DIRTY_DDB) {
3583 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3584 val = I915_READ(WM_MISC);
3585 if (results->partitioning == INTEL_DDB_PART_1_2)
3586 val &= ~WM_MISC_DATA_PARTITION_5_6;
3588 val |= WM_MISC_DATA_PARTITION_5_6;
3589 I915_WRITE(WM_MISC, val);
3591 val = I915_READ(DISP_ARB_CTL2);
3592 if (results->partitioning == INTEL_DDB_PART_1_2)
3593 val &= ~DISP_DATA_PARTITION_5_6;
3595 val |= DISP_DATA_PARTITION_5_6;
3596 I915_WRITE(DISP_ARB_CTL2, val);
3600 if (dirty & WM_DIRTY_FBC) {
3601 val = I915_READ(DISP_ARB_CTL);
3602 if (results->enable_fbc_wm)
3603 val &= ~DISP_FBC_WM_DIS;
3605 val |= DISP_FBC_WM_DIS;
3606 I915_WRITE(DISP_ARB_CTL, val);
3609 if (dirty & WM_DIRTY_LP(1) &&
3610 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3611 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3613 if (INTEL_GEN(dev_priv) >= 7) {
3614 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3615 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3616 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3617 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3620 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3621 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3622 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3623 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3624 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3625 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3627 dev_priv->wm.hw = *results;
3630 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3632 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3635 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3638 int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3639 u8 enabled_slices_mask = 0;
3641 for (i = 0; i < max_slices; i++) {
3642 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3643 enabled_slices_mask |= BIT(i);
3646 return enabled_slices_mask;
3650 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3651 * so assume we'll always need it in order to avoid underruns.
3653 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3655 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3659 intel_has_sagv(struct drm_i915_private *dev_priv)
3661 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3662 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3666 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3668 if (INTEL_GEN(dev_priv) >= 12) {
3672 ret = sandybridge_pcode_read(dev_priv,
3673 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3676 dev_priv->sagv_block_time_us = val;
3680 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3681 } else if (IS_GEN(dev_priv, 11)) {
3682 dev_priv->sagv_block_time_us = 10;
3684 } else if (IS_GEN(dev_priv, 10)) {
3685 dev_priv->sagv_block_time_us = 20;
3687 } else if (IS_GEN(dev_priv, 9)) {
3688 dev_priv->sagv_block_time_us = 30;
3691 MISSING_CASE(INTEL_GEN(dev_priv));
3694 /* Default to an unusable block time */
3695 dev_priv->sagv_block_time_us = -1;
3699 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3700 * depending on power and performance requirements. The display engine access
3701 * to system memory is blocked during the adjustment time. Because of the
3702 * blocking time, having this enabled can cause full system hangs and/or pipe
3703 * underruns if we don't meet all of the following requirements:
3705 * - <= 1 pipe enabled
3706 * - All planes can enable watermarks for latencies >= SAGV engine block time
3707 * - We're not using an interlaced display configuration
3710 intel_enable_sagv(struct drm_i915_private *dev_priv)
3714 if (!intel_has_sagv(dev_priv))
3717 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3720 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3721 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3724 /* We don't need to wait for SAGV when enabling */
3727 * Some skl systems, pre-release machines in particular,
3728 * don't actually have SAGV.
3730 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3731 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3732 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3734 } else if (ret < 0) {
3735 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3739 dev_priv->sagv_status = I915_SAGV_ENABLED;
3744 intel_disable_sagv(struct drm_i915_private *dev_priv)
3748 if (!intel_has_sagv(dev_priv))
3751 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3754 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3755 /* bspec says to keep retrying for at least 1 ms */
3756 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3758 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3761 * Some skl systems, pre-release machines in particular,
3762 * don't actually have SAGV.
3764 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3765 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3766 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3768 } else if (ret < 0) {
3769 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3773 dev_priv->sagv_status = I915_SAGV_DISABLED;
3777 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3779 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3780 const struct intel_bw_state *new_bw_state;
3781 const struct intel_bw_state *old_bw_state;
3785 * Just return if we can't control SAGV or don't have it.
3786 * This is different from situation when we have SAGV but just can't
3787 * afford it due to DBuf limitation - in case if SAGV is completely
3788 * disabled in a BIOS, we are not even allowed to send a PCode request,
3789 * as it will throw an error. So have to check it here.
3791 if (!intel_has_sagv(dev_priv))
3794 new_bw_state = intel_atomic_get_new_bw_state(state);
3798 if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3799 intel_disable_sagv(dev_priv);
3803 old_bw_state = intel_atomic_get_old_bw_state(state);
3807 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3810 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3813 * If new mask is zero - means there is nothing to mask,
3814 * we can only unmask, which should be done in unmask.
3820 * Restrict required qgv points before updating the configuration.
3821 * According to BSpec we can't mask and unmask qgv points at the same
3822 * time. Also masking should be done before updating the configuration
3823 * and unmasking afterwards.
3825 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3828 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3830 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3831 const struct intel_bw_state *new_bw_state;
3832 const struct intel_bw_state *old_bw_state;
3836 * Just return if we can't control SAGV or don't have it.
3837 * This is different from situation when we have SAGV but just can't
3838 * afford it due to DBuf limitation - in case if SAGV is completely
3839 * disabled in a BIOS, we are not even allowed to send a PCode request,
3840 * as it will throw an error. So have to check it here.
3842 if (!intel_has_sagv(dev_priv))
3845 new_bw_state = intel_atomic_get_new_bw_state(state);
3849 if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3850 intel_enable_sagv(dev_priv);
3854 old_bw_state = intel_atomic_get_old_bw_state(state);
3858 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3861 new_mask = new_bw_state->qgv_points_mask;
3864 * Allow required qgv points after updating the configuration.
3865 * According to BSpec we can't mask and unmask qgv points at the same
3866 * time. Also masking should be done before updating the configuration
3867 * and unmasking afterwards.
3869 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3872 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3874 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3875 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3876 struct intel_plane *plane;
3877 const struct intel_plane_state *plane_state;
3880 if (!intel_has_sagv(dev_priv))
3883 if (!crtc_state->hw.active)
3886 if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3889 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3890 const struct skl_plane_wm *wm =
3891 &crtc_state->wm.skl.optimal.planes[plane->id];
3893 /* Skip this plane if it's not enabled */
3894 if (!wm->wm[0].plane_en)
3897 /* Find the highest enabled wm level for this plane */
3898 for (level = ilk_wm_max_level(dev_priv);
3899 !wm->wm[level].plane_en; --level)
3902 latency = dev_priv->wm.skl_latency[level];
3904 if (skl_needs_memory_bw_wa(dev_priv) &&
3905 plane_state->uapi.fb->modifier ==
3906 I915_FORMAT_MOD_X_TILED)
3910 * If any of the planes on this pipe don't enable wm levels that
3911 * incur memory latencies higher than sagv_block_time_us we
3912 * can't enable SAGV.
3914 if (latency < dev_priv->sagv_block_time_us)
3921 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3923 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3924 enum plane_id plane_id;
3926 if (!crtc_state->hw.active)
3929 for_each_plane_id_on_crtc(crtc, plane_id) {
3930 const struct skl_ddb_entry *plane_alloc =
3931 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3932 const struct skl_plane_wm *wm =
3933 &crtc_state->wm.skl.optimal.planes[plane_id];
3935 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3942 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3944 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3945 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3947 if (INTEL_GEN(dev_priv) >= 12)
3948 return tgl_crtc_can_enable_sagv(crtc_state);
3950 return skl_crtc_can_enable_sagv(crtc_state);
3953 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3954 const struct intel_bw_state *bw_state)
3956 if (INTEL_GEN(dev_priv) < 11 &&
3957 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3960 return bw_state->pipe_sagv_reject == 0;
3963 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3965 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3967 struct intel_crtc *crtc;
3968 struct intel_crtc_state *new_crtc_state;
3969 struct intel_bw_state *new_bw_state = NULL;
3970 const struct intel_bw_state *old_bw_state = NULL;
3973 for_each_new_intel_crtc_in_state(state, crtc,
3974 new_crtc_state, i) {
3975 new_bw_state = intel_atomic_get_bw_state(state);
3976 if (IS_ERR(new_bw_state))
3977 return PTR_ERR(new_bw_state);
3979 old_bw_state = intel_atomic_get_old_bw_state(state);
3981 if (intel_crtc_can_enable_sagv(new_crtc_state))
3982 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3984 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3990 new_bw_state->active_pipes =
3991 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3993 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3994 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3999 for_each_new_intel_crtc_in_state(state, crtc,
4000 new_crtc_state, i) {
4001 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4004 * We store use_sagv_wm in the crtc state rather than relying on
4005 * that bw state since we have no convenient way to get at the
4006 * latter from the plane commit hooks (especially in the legacy
4009 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4010 intel_can_enable_sagv(dev_priv, new_bw_state);
4013 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4014 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4015 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4018 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4019 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4028 * Calculate initial DBuf slice offset, based on slice size
4029 * and mask(i.e if slice size is 1024 and second slice is enabled
4030 * offset would be 1024)
4033 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4037 unsigned int offset = 0;
4039 if (!dbuf_slice_mask)
4042 offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4044 WARN_ON(offset >= ddb_size);
4048 u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4050 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4051 drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4053 if (INTEL_GEN(dev_priv) < 11)
4054 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4059 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4060 const struct skl_ddb_entry *entry)
4063 u16 ddb_size = intel_get_ddb_size(dev_priv);
4064 u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4065 u16 slice_size = ddb_size / num_supported_slices;
4069 if (!skl_ddb_entry_size(entry))
4072 start_slice = entry->start / slice_size;
4073 end_slice = (entry->end - 1) / slice_size;
4076 * Per plane DDB entry can in a really worst case be on multiple slices
4077 * but single entry is anyway contigious.
4079 while (start_slice <= end_slice) {
4080 slice_mask |= BIT(start_slice);
4087 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4091 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4092 const struct intel_crtc_state *crtc_state,
4093 const u64 total_data_rate,
4094 struct skl_ddb_entry *alloc, /* out */
4095 int *num_active /* out */)
4097 struct drm_atomic_state *state = crtc_state->uapi.state;
4098 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4099 struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4100 const struct intel_crtc *crtc;
4101 u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4102 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4103 struct intel_dbuf_state *new_dbuf_state =
4104 intel_atomic_get_new_dbuf_state(intel_state);
4105 const struct intel_dbuf_state *old_dbuf_state =
4106 intel_atomic_get_old_dbuf_state(intel_state);
4107 u8 active_pipes = new_dbuf_state->active_pipes;
4111 u32 dbuf_slice_mask;
4114 u32 total_slice_mask;
4118 *num_active = hweight8(active_pipes);
4120 if (!crtc_state->hw.active) {
4126 ddb_size = intel_get_ddb_size(dev_priv);
4128 slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4131 * If the state doesn't change the active CRTC's or there is no
4132 * modeset request, then there's no need to recalculate;
4133 * the existing pipe allocation limits should remain unchanged.
4134 * Note that we're safe from racing commits since any racing commit
4135 * that changes the active CRTC list or do modeset would need to
4136 * grab _all_ crtc locks, including the one we currently hold.
4138 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes &&
4139 !dev_priv->wm.distrust_bios_wm) {
4141 * alloc may be cleared by clear_intel_crtc_state,
4142 * copy from old state to be sure
4144 * FIXME get rid of this mess
4146 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4151 * Get allowed DBuf slices for correspondent pipe and platform.
4153 dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4156 * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4157 * and slice size is 1024, the offset would be 1024
4159 offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4160 slice_size, ddb_size);
4163 * Figure out total size of allowed DBuf slices, which is basically
4164 * a number of allowed slices for that pipe multiplied by slice size.
4166 * range ddb entries are still allocated in proportion to display width.
4168 ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4171 * Watermark/ddb requirement highly depends upon width of the
4172 * framebuffer, So instead of allocating DDB equally among pipes
4173 * distribute DDB based on resolution/width of the display.
4175 total_slice_mask = dbuf_slice_mask;
4176 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4177 const struct drm_display_mode *adjusted_mode =
4178 &crtc_state->hw.adjusted_mode;
4179 enum pipe pipe = crtc->pipe;
4180 int hdisplay, vdisplay;
4181 u32 pipe_dbuf_slice_mask;
4183 if (!crtc_state->hw.active)
4186 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4190 * According to BSpec pipe can share one dbuf slice with another
4191 * pipes or pipe can use multiple dbufs, in both cases we
4192 * account for other pipes only if they have exactly same mask.
4193 * However we need to account how many slices we should enable
4196 total_slice_mask |= pipe_dbuf_slice_mask;
4199 * Do not account pipes using other slice sets
4200 * luckily as of current BSpec slice sets do not partially
4201 * intersect(pipes share either same one slice or same slice set
4202 * i.e no partial intersection), so it is enough to check for
4205 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4208 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4210 total_width_in_range += hdisplay;
4212 if (pipe < for_pipe)
4213 width_before_pipe_in_range += hdisplay;
4214 else if (pipe == for_pipe)
4215 pipe_width = hdisplay;
4219 * FIXME: For now we always enable slice S1 as per
4220 * the Bspec display initialization sequence.
4222 new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1);
4224 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4225 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4230 start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4231 end = ddb_range_size *
4232 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4234 alloc->start = offset + start;
4235 alloc->end = offset + end;
4237 drm_dbg_kms(&dev_priv->drm,
4238 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
4239 for_crtc->base.id, for_crtc->name,
4240 dbuf_slice_mask, alloc->start, alloc->end, active_pipes);
4245 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4246 int width, const struct drm_format_info *format,
4247 u64 modifier, unsigned int rotation,
4248 u32 plane_pixel_rate, struct skl_wm_params *wp,
4250 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4252 unsigned int latency,
4253 const struct skl_wm_params *wp,
4254 const struct skl_wm_level *result_prev,
4255 struct skl_wm_level *result /* out */);
4258 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4261 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4262 int level, max_level = ilk_wm_max_level(dev_priv);
4263 struct skl_wm_level wm = {};
4264 int ret, min_ddb_alloc = 0;
4265 struct skl_wm_params wp;
4267 ret = skl_compute_wm_params(crtc_state, 256,
4268 drm_format_info(DRM_FORMAT_ARGB8888),
4269 DRM_FORMAT_MOD_LINEAR,
4271 crtc_state->pixel_rate, &wp, 0);
4272 drm_WARN_ON(&dev_priv->drm, ret);
4274 for (level = 0; level <= max_level; level++) {
4275 unsigned int latency = dev_priv->wm.skl_latency[level];
4277 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4278 if (wm.min_ddb_alloc == U16_MAX)
4281 min_ddb_alloc = wm.min_ddb_alloc;
4284 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4287 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4288 struct skl_ddb_entry *entry, u32 reg)
4291 entry->start = reg & DDB_ENTRY_MASK;
4292 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4299 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4300 const enum pipe pipe,
4301 const enum plane_id plane_id,
4302 struct skl_ddb_entry *ddb_y,
4303 struct skl_ddb_entry *ddb_uv)
4308 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4309 if (plane_id == PLANE_CURSOR) {
4310 val = I915_READ(CUR_BUF_CFG(pipe));
4311 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4315 val = I915_READ(PLANE_CTL(pipe, plane_id));
4317 /* No DDB allocated for disabled planes */
4318 if (val & PLANE_CTL_ENABLE)
4319 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4320 val & PLANE_CTL_ORDER_RGBX,
4321 val & PLANE_CTL_ALPHA_MASK);
4323 if (INTEL_GEN(dev_priv) >= 11) {
4324 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4325 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4327 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4328 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4331 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4334 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4335 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4339 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4340 struct skl_ddb_entry *ddb_y,
4341 struct skl_ddb_entry *ddb_uv)
4343 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4344 enum intel_display_power_domain power_domain;
4345 enum pipe pipe = crtc->pipe;
4346 intel_wakeref_t wakeref;
4347 enum plane_id plane_id;
4349 power_domain = POWER_DOMAIN_PIPE(pipe);
4350 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4354 for_each_plane_id_on_crtc(crtc, plane_id)
4355 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4360 intel_display_power_put(dev_priv, power_domain, wakeref);
4364 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4365 * The bspec defines downscale amount as:
4368 * Horizontal down scale amount = maximum[1, Horizontal source size /
4369 * Horizontal destination size]
4370 * Vertical down scale amount = maximum[1, Vertical source size /
4371 * Vertical destination size]
4372 * Total down scale amount = Horizontal down scale amount *
4373 * Vertical down scale amount
4376 * Return value is provided in 16.16 fixed point form to retain fractional part.
4377 * Caller should take care of dividing & rounding off the value.
4379 static uint_fixed_16_16_t
4380 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4381 const struct intel_plane_state *plane_state)
4383 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4384 u32 src_w, src_h, dst_w, dst_h;
4385 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4386 uint_fixed_16_16_t downscale_h, downscale_w;
4388 if (drm_WARN_ON(&dev_priv->drm,
4389 !intel_wm_plane_visible(crtc_state, plane_state)))
4390 return u32_to_fixed16(0);
4393 * Src coordinates are already rotated by 270 degrees for
4394 * the 90/270 degree plane rotation cases (to match the
4395 * GTT mapping), hence no need to account for rotation here.
4397 * n.b., src is 16.16 fixed point, dst is whole integer.
4399 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4400 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4401 dst_w = drm_rect_width(&plane_state->uapi.dst);
4402 dst_h = drm_rect_height(&plane_state->uapi.dst);
4404 fp_w_ratio = div_fixed16(src_w, dst_w);
4405 fp_h_ratio = div_fixed16(src_h, dst_h);
4406 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4407 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4409 return mul_fixed16(downscale_w, downscale_h);
4412 struct dbuf_slice_conf_entry {
4414 u8 dbuf_mask[I915_MAX_PIPES];
4418 * Table taken from Bspec 12716
4419 * Pipes do have some preferred DBuf slice affinity,
4420 * plus there are some hardcoded requirements on how
4421 * those should be distributed for multipipe scenarios.
4422 * For more DBuf slices algorithm can get even more messy
4423 * and less readable, so decided to use a table almost
4424 * as is from BSpec itself - that way it is at least easier
4425 * to compare, change and check.
4427 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4428 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4431 .active_pipes = BIT(PIPE_A),
4433 [PIPE_A] = BIT(DBUF_S1),
4437 .active_pipes = BIT(PIPE_B),
4439 [PIPE_B] = BIT(DBUF_S1),
4443 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4445 [PIPE_A] = BIT(DBUF_S1),
4446 [PIPE_B] = BIT(DBUF_S2),
4450 .active_pipes = BIT(PIPE_C),
4452 [PIPE_C] = BIT(DBUF_S2),
4456 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4458 [PIPE_A] = BIT(DBUF_S1),
4459 [PIPE_C] = BIT(DBUF_S2),
4463 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4465 [PIPE_B] = BIT(DBUF_S1),
4466 [PIPE_C] = BIT(DBUF_S2),
4470 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4472 [PIPE_A] = BIT(DBUF_S1),
4473 [PIPE_B] = BIT(DBUF_S1),
4474 [PIPE_C] = BIT(DBUF_S2),
4481 * Table taken from Bspec 49255
4482 * Pipes do have some preferred DBuf slice affinity,
4483 * plus there are some hardcoded requirements on how
4484 * those should be distributed for multipipe scenarios.
4485 * For more DBuf slices algorithm can get even more messy
4486 * and less readable, so decided to use a table almost
4487 * as is from BSpec itself - that way it is at least easier
4488 * to compare, change and check.
4490 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4491 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4494 .active_pipes = BIT(PIPE_A),
4496 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4500 .active_pipes = BIT(PIPE_B),
4502 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4506 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4508 [PIPE_A] = BIT(DBUF_S2),
4509 [PIPE_B] = BIT(DBUF_S1),
4513 .active_pipes = BIT(PIPE_C),
4515 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4519 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4521 [PIPE_A] = BIT(DBUF_S1),
4522 [PIPE_C] = BIT(DBUF_S2),
4526 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4528 [PIPE_B] = BIT(DBUF_S1),
4529 [PIPE_C] = BIT(DBUF_S2),
4533 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4535 [PIPE_A] = BIT(DBUF_S1),
4536 [PIPE_B] = BIT(DBUF_S1),
4537 [PIPE_C] = BIT(DBUF_S2),
4541 .active_pipes = BIT(PIPE_D),
4543 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4547 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4549 [PIPE_A] = BIT(DBUF_S1),
4550 [PIPE_D] = BIT(DBUF_S2),
4554 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4556 [PIPE_B] = BIT(DBUF_S1),
4557 [PIPE_D] = BIT(DBUF_S2),
4561 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4563 [PIPE_A] = BIT(DBUF_S1),
4564 [PIPE_B] = BIT(DBUF_S1),
4565 [PIPE_D] = BIT(DBUF_S2),
4569 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4571 [PIPE_C] = BIT(DBUF_S1),
4572 [PIPE_D] = BIT(DBUF_S2),
4576 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4578 [PIPE_A] = BIT(DBUF_S1),
4579 [PIPE_C] = BIT(DBUF_S2),
4580 [PIPE_D] = BIT(DBUF_S2),
4584 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4586 [PIPE_B] = BIT(DBUF_S1),
4587 [PIPE_C] = BIT(DBUF_S2),
4588 [PIPE_D] = BIT(DBUF_S2),
4592 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4594 [PIPE_A] = BIT(DBUF_S1),
4595 [PIPE_B] = BIT(DBUF_S1),
4596 [PIPE_C] = BIT(DBUF_S2),
4597 [PIPE_D] = BIT(DBUF_S2),
4603 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4604 const struct dbuf_slice_conf_entry *dbuf_slices)
4608 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4609 if (dbuf_slices[i].active_pipes == active_pipes)
4610 return dbuf_slices[i].dbuf_mask[pipe];
4616 * This function finds an entry with same enabled pipe configuration and
4617 * returns correspondent DBuf slice mask as stated in BSpec for particular
4620 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4623 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4624 * required calculating "pipe ratio" in order to determine
4625 * if one or two slices can be used for single pipe configurations
4626 * as additional constraint to the existing table.
4627 * However based on recent info, it should be not "pipe ratio"
4628 * but rather ratio between pixel_rate and cdclk with additional
4629 * constants, so for now we are using only table until this is
4630 * clarified. Also this is the reason why crtc_state param is
4631 * still here - we will need it once those additional constraints
4634 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4637 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4639 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4642 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4645 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4646 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4647 enum pipe pipe = crtc->pipe;
4649 if (IS_GEN(dev_priv, 12))
4650 return tgl_compute_dbuf_slices(pipe, active_pipes);
4651 else if (IS_GEN(dev_priv, 11))
4652 return icl_compute_dbuf_slices(pipe, active_pipes);
4654 * For anything else just return one slice yet.
4655 * Should be extended for other platforms.
4657 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4661 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4662 const struct intel_plane_state *plane_state,
4665 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4666 const struct drm_framebuffer *fb = plane_state->hw.fb;
4668 u32 width = 0, height = 0;
4669 uint_fixed_16_16_t down_scale_amount;
4672 if (!plane_state->uapi.visible)
4675 if (plane->id == PLANE_CURSOR)
4678 if (color_plane == 1 &&
4679 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4683 * Src coordinates are already rotated by 270 degrees for
4684 * the 90/270 degree plane rotation cases (to match the
4685 * GTT mapping), hence no need to account for rotation here.
4687 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4688 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4690 /* UV plane does 1/2 pixel sub-sampling */
4691 if (color_plane == 1) {
4696 data_rate = width * height;
4698 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4700 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4702 rate *= fb->format->cpp[color_plane];
4707 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4708 u64 *plane_data_rate,
4709 u64 *uv_plane_data_rate)
4711 struct intel_plane *plane;
4712 const struct intel_plane_state *plane_state;
4713 u64 total_data_rate = 0;
4715 /* Calculate and cache data rate for each plane */
4716 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4717 enum plane_id plane_id = plane->id;
4721 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4722 plane_data_rate[plane_id] = rate;
4723 total_data_rate += rate;
4726 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4727 uv_plane_data_rate[plane_id] = rate;
4728 total_data_rate += rate;
4731 return total_data_rate;
4735 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4736 u64 *plane_data_rate)
4738 struct intel_plane *plane;
4739 const struct intel_plane_state *plane_state;
4740 u64 total_data_rate = 0;
4742 /* Calculate and cache data rate for each plane */
4743 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4744 enum plane_id plane_id = plane->id;
4747 if (!plane_state->planar_linked_plane) {
4748 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4749 plane_data_rate[plane_id] = rate;
4750 total_data_rate += rate;
4752 enum plane_id y_plane_id;
4755 * The slave plane might not iterate in
4756 * intel_atomic_crtc_state_for_each_plane_state(),
4757 * and needs the master plane state which may be
4758 * NULL if we try get_new_plane_state(), so we
4759 * always calculate from the master.
4761 if (plane_state->planar_slave)
4764 /* Y plane rate is calculated on the slave */
4765 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4766 y_plane_id = plane_state->planar_linked_plane->id;
4767 plane_data_rate[y_plane_id] = rate;
4768 total_data_rate += rate;
4770 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4771 plane_data_rate[plane_id] = rate;
4772 total_data_rate += rate;
4776 return total_data_rate;
4779 static const struct skl_wm_level *
4780 skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4781 enum plane_id plane_id,
4784 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4785 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4787 if (level == 0 && pipe_wm->use_sagv_wm)
4788 return &wm->sagv_wm0;
4790 return &wm->wm[level];
4794 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4796 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4797 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4798 struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4799 u16 alloc_size, start = 0;
4800 u16 total[I915_MAX_PLANES] = {};
4801 u16 uv_total[I915_MAX_PLANES] = {};
4802 u64 total_data_rate;
4803 enum plane_id plane_id;
4805 u64 plane_data_rate[I915_MAX_PLANES] = {};
4806 u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4811 /* Clear the partitioning for disabled planes. */
4812 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4813 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4815 if (!crtc_state->hw.active) {
4816 struct intel_atomic_state *state =
4817 to_intel_atomic_state(crtc_state->uapi.state);
4818 struct intel_dbuf_state *new_dbuf_state =
4819 intel_atomic_get_new_dbuf_state(state);
4820 const struct intel_dbuf_state *old_dbuf_state =
4821 intel_atomic_get_old_dbuf_state(state);
4824 * FIXME hack to make sure we compute this sensibly when
4825 * turning off all the pipes. Otherwise we leave it at
4826 * whatever we had previously, and then runtime PM will
4827 * mess it up by turning off all but S1. Remove this
4828 * once the dbuf state computation flow becomes sane.
4830 if (new_dbuf_state->active_pipes == 0) {
4831 new_dbuf_state->enabled_slices = BIT(DBUF_S1);
4833 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4834 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4840 alloc->start = alloc->end = 0;
4844 if (INTEL_GEN(dev_priv) >= 11)
4846 icl_get_total_relative_data_rate(crtc_state,
4850 skl_get_total_relative_data_rate(crtc_state,
4852 uv_plane_data_rate);
4854 ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state,
4856 alloc, &num_active);
4860 alloc_size = skl_ddb_entry_size(alloc);
4861 if (alloc_size == 0)
4864 /* Allocate fixed number of blocks for cursor. */
4865 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4866 alloc_size -= total[PLANE_CURSOR];
4867 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4868 alloc->end - total[PLANE_CURSOR];
4869 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4871 if (total_data_rate == 0)
4875 * Find the highest watermark level for which we can satisfy the block
4876 * requirement of active planes.
4878 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4880 for_each_plane_id_on_crtc(crtc, plane_id) {
4881 const struct skl_plane_wm *wm =
4882 &crtc_state->wm.skl.optimal.planes[plane_id];
4884 if (plane_id == PLANE_CURSOR) {
4885 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4886 drm_WARN_ON(&dev_priv->drm,
4887 wm->wm[level].min_ddb_alloc != U16_MAX);
4894 blocks += wm->wm[level].min_ddb_alloc;
4895 blocks += wm->uv_wm[level].min_ddb_alloc;
4898 if (blocks <= alloc_size) {
4899 alloc_size -= blocks;
4905 drm_dbg_kms(&dev_priv->drm,
4906 "Requested display configuration exceeds system DDB limitations");
4907 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4908 blocks, alloc_size);
4913 * Grant each plane the blocks it requires at the highest achievable
4914 * watermark level, plus an extra share of the leftover blocks
4915 * proportional to its relative data rate.
4917 for_each_plane_id_on_crtc(crtc, plane_id) {
4918 const struct skl_plane_wm *wm =
4919 &crtc_state->wm.skl.optimal.planes[plane_id];
4923 if (plane_id == PLANE_CURSOR)
4927 * We've accounted for all active planes; remaining planes are
4930 if (total_data_rate == 0)
4933 rate = plane_data_rate[plane_id];
4934 extra = min_t(u16, alloc_size,
4935 DIV64_U64_ROUND_UP(alloc_size * rate,
4937 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4938 alloc_size -= extra;
4939 total_data_rate -= rate;
4941 if (total_data_rate == 0)
4944 rate = uv_plane_data_rate[plane_id];
4945 extra = min_t(u16, alloc_size,
4946 DIV64_U64_ROUND_UP(alloc_size * rate,
4948 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4949 alloc_size -= extra;
4950 total_data_rate -= rate;
4952 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4954 /* Set the actual DDB start/end points for each plane */
4955 start = alloc->start;
4956 for_each_plane_id_on_crtc(crtc, plane_id) {
4957 struct skl_ddb_entry *plane_alloc =
4958 &crtc_state->wm.skl.plane_ddb_y[plane_id];
4959 struct skl_ddb_entry *uv_plane_alloc =
4960 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4962 if (plane_id == PLANE_CURSOR)
4965 /* Gen11+ uses a separate plane for UV watermarks */
4966 drm_WARN_ON(&dev_priv->drm,
4967 INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4969 /* Leave disabled planes at (0,0) */
4970 if (total[plane_id]) {
4971 plane_alloc->start = start;
4972 start += total[plane_id];
4973 plane_alloc->end = start;
4976 if (uv_total[plane_id]) {
4977 uv_plane_alloc->start = start;
4978 start += uv_total[plane_id];
4979 uv_plane_alloc->end = start;
4984 * When we calculated watermark values we didn't know how high
4985 * of a level we'd actually be able to hit, so we just marked
4986 * all levels as "enabled." Go back now and disable the ones
4987 * that aren't actually possible.
4989 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4990 for_each_plane_id_on_crtc(crtc, plane_id) {
4991 struct skl_plane_wm *wm =
4992 &crtc_state->wm.skl.optimal.planes[plane_id];
4995 * We only disable the watermarks for each plane if
4996 * they exceed the ddb allocation of said plane. This
4997 * is done so that we don't end up touching cursor
4998 * watermarks needlessly when some other plane reduces
4999 * our max possible watermark level.
5001 * Bspec has this to say about the PLANE_WM enable bit:
5002 * "All the watermarks at this level for all enabled
5003 * planes must be enabled before the level will be used."
5004 * So this is actually safe to do.
5006 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
5007 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
5008 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
5011 * Wa_1408961008:icl, ehl
5012 * Underruns with WM1+ disabled
5014 if (IS_GEN(dev_priv, 11) &&
5015 level == 1 && wm->wm[0].plane_en) {
5016 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
5017 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
5018 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5024 * Go back and disable the transition watermark if it turns out we
5025 * don't have enough DDB blocks for it.
5027 for_each_plane_id_on_crtc(crtc, plane_id) {
5028 struct skl_plane_wm *wm =
5029 &crtc_state->wm.skl.optimal.planes[plane_id];
5031 if (wm->trans_wm.plane_res_b >= total[plane_id])
5032 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
5039 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5040 * for the read latency) and cpp should always be <= 8, so that
5041 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5042 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5044 static uint_fixed_16_16_t
5045 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5046 u8 cpp, u32 latency, u32 dbuf_block_size)
5048 u32 wm_intermediate_val;
5049 uint_fixed_16_16_t ret;
5052 return FP_16_16_MAX;
5054 wm_intermediate_val = latency * pixel_rate * cpp;
5055 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5057 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5058 ret = add_fixed16_u32(ret, 1);
5063 static uint_fixed_16_16_t
5064 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5065 uint_fixed_16_16_t plane_blocks_per_line)
5067 u32 wm_intermediate_val;
5068 uint_fixed_16_16_t ret;
5071 return FP_16_16_MAX;
5073 wm_intermediate_val = latency * pixel_rate;
5074 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5075 pipe_htotal * 1000);
5076 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5080 static uint_fixed_16_16_t
5081 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5083 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5086 uint_fixed_16_16_t linetime_us;
5088 if (!crtc_state->hw.active)
5089 return u32_to_fixed16(0);
5091 pixel_rate = crtc_state->pixel_rate;
5093 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5094 return u32_to_fixed16(0);
5096 crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5097 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5103 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5104 const struct intel_plane_state *plane_state)
5106 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5107 u64 adjusted_pixel_rate;
5108 uint_fixed_16_16_t downscale_amount;
5110 /* Shouldn't reach here on disabled planes... */
5111 if (drm_WARN_ON(&dev_priv->drm,
5112 !intel_wm_plane_visible(crtc_state, plane_state)))
5116 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5117 * with additional adjustments for plane-specific scaling.
5119 adjusted_pixel_rate = crtc_state->pixel_rate;
5120 downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5122 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5127 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5128 int width, const struct drm_format_info *format,
5129 u64 modifier, unsigned int rotation,
5130 u32 plane_pixel_rate, struct skl_wm_params *wp,
5133 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5134 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5137 /* only planar format has two planes */
5138 if (color_plane == 1 &&
5139 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5140 drm_dbg_kms(&dev_priv->drm,
5141 "Non planar format have single plane\n");
5145 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5146 modifier == I915_FORMAT_MOD_Yf_TILED ||
5147 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5148 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5149 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5150 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5151 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5152 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5155 if (color_plane == 1 && wp->is_planar)
5158 wp->cpp = format->cpp[color_plane];
5159 wp->plane_pixel_rate = plane_pixel_rate;
5161 if (INTEL_GEN(dev_priv) >= 11 &&
5162 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5163 wp->dbuf_block_size = 256;
5165 wp->dbuf_block_size = 512;
5167 if (drm_rotation_90_or_270(rotation)) {
5170 wp->y_min_scanlines = 16;
5173 wp->y_min_scanlines = 8;
5176 wp->y_min_scanlines = 4;
5179 MISSING_CASE(wp->cpp);
5183 wp->y_min_scanlines = 4;
5186 if (skl_needs_memory_bw_wa(dev_priv))
5187 wp->y_min_scanlines *= 2;
5189 wp->plane_bytes_per_line = wp->width * wp->cpp;
5191 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5192 wp->y_min_scanlines,
5193 wp->dbuf_block_size);
5195 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5198 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5199 wp->y_min_scanlines);
5201 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5202 wp->dbuf_block_size);
5205 INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5208 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5211 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5212 wp->plane_blocks_per_line);
5214 wp->linetime_us = fixed16_to_u32_round_up(
5215 intel_get_linetime_us(crtc_state));
5221 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5222 const struct intel_plane_state *plane_state,
5223 struct skl_wm_params *wp, int color_plane)
5225 const struct drm_framebuffer *fb = plane_state->hw.fb;
5229 * Src coordinates are already rotated by 270 degrees for
5230 * the 90/270 degree plane rotation cases (to match the
5231 * GTT mapping), hence no need to account for rotation here.
5233 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5235 return skl_compute_wm_params(crtc_state, width,
5236 fb->format, fb->modifier,
5237 plane_state->hw.rotation,
5238 skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5242 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5244 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5247 /* The number of lines are ignored for the level 0 watermark. */
5251 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5253 unsigned int latency,
5254 const struct skl_wm_params *wp,
5255 const struct skl_wm_level *result_prev,
5256 struct skl_wm_level *result /* out */)
5258 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5259 uint_fixed_16_16_t method1, method2;
5260 uint_fixed_16_16_t selected_result;
5261 u32 res_blocks, res_lines, min_ddb_alloc = 0;
5265 result->min_ddb_alloc = U16_MAX;
5270 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5271 * Display WA #1141: kbl,cfl
5273 if ((IS_KABYLAKE(dev_priv) ||
5274 IS_COFFEELAKE(dev_priv) ||
5275 IS_COMETLAKE(dev_priv)) &&
5276 dev_priv->ipc_enabled)
5279 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5282 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5283 wp->cpp, latency, wp->dbuf_block_size);
5284 method2 = skl_wm_method2(wp->plane_pixel_rate,
5285 crtc_state->hw.adjusted_mode.crtc_htotal,
5287 wp->plane_blocks_per_line);
5290 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5292 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5293 wp->dbuf_block_size < 1) &&
5294 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5295 selected_result = method2;
5296 } else if (latency >= wp->linetime_us) {
5297 if (IS_GEN(dev_priv, 9) &&
5298 !IS_GEMINILAKE(dev_priv))
5299 selected_result = min_fixed16(method1, method2);
5301 selected_result = method2;
5303 selected_result = method1;
5307 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5308 res_lines = div_round_up_fixed16(selected_result,
5309 wp->plane_blocks_per_line);
5311 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5312 /* Display WA #1125: skl,bxt,kbl */
5313 if (level == 0 && wp->rc_surface)
5315 fixed16_to_u32_round_up(wp->y_tile_minimum);
5317 /* Display WA #1126: skl,bxt,kbl */
5318 if (level >= 1 && level <= 7) {
5321 fixed16_to_u32_round_up(wp->y_tile_minimum);
5322 res_lines += wp->y_min_scanlines;
5328 * Make sure result blocks for higher latency levels are
5329 * atleast as high as level below the current level.
5330 * Assumption in DDB algorithm optimization for special
5331 * cases. Also covers Display WA #1125 for RC.
5333 if (result_prev->plane_res_b > res_blocks)
5334 res_blocks = result_prev->plane_res_b;
5338 if (INTEL_GEN(dev_priv) >= 11) {
5342 if (res_lines % wp->y_min_scanlines == 0)
5343 extra_lines = wp->y_min_scanlines;
5345 extra_lines = wp->y_min_scanlines * 2 -
5346 res_lines % wp->y_min_scanlines;
5348 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5349 wp->plane_blocks_per_line);
5351 min_ddb_alloc = res_blocks +
5352 DIV_ROUND_UP(res_blocks, 10);
5356 if (!skl_wm_has_lines(dev_priv, level))
5359 if (res_lines > 31) {
5361 result->min_ddb_alloc = U16_MAX;
5366 * If res_lines is valid, assume we can use this watermark level
5367 * for now. We'll come back and disable it after we calculate the
5368 * DDB allocation if it turns out we don't actually have enough
5369 * blocks to satisfy it.
5371 result->plane_res_b = res_blocks;
5372 result->plane_res_l = res_lines;
5373 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5374 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5375 result->plane_en = true;
5379 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5380 const struct skl_wm_params *wm_params,
5381 struct skl_wm_level *levels)
5383 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5384 int level, max_level = ilk_wm_max_level(dev_priv);
5385 struct skl_wm_level *result_prev = &levels[0];
5387 for (level = 0; level <= max_level; level++) {
5388 struct skl_wm_level *result = &levels[level];
5389 unsigned int latency = dev_priv->wm.skl_latency[level];
5391 skl_compute_plane_wm(crtc_state, level, latency,
5392 wm_params, result_prev, result);
5394 result_prev = result;
5398 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5399 const struct skl_wm_params *wm_params,
5400 struct skl_plane_wm *plane_wm)
5402 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5403 struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5404 struct skl_wm_level *levels = plane_wm->wm;
5405 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5407 skl_compute_plane_wm(crtc_state, 0, latency,
5408 wm_params, &levels[0],
5412 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5413 const struct skl_wm_params *wp,
5414 struct skl_plane_wm *wm)
5416 struct drm_device *dev = crtc_state->uapi.crtc->dev;
5417 const struct drm_i915_private *dev_priv = to_i915(dev);
5418 u16 trans_min, trans_amount, trans_y_tile_min;
5419 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5421 /* Transition WM don't make any sense if ipc is disabled */
5422 if (!dev_priv->ipc_enabled)
5426 * WaDisableTWM:skl,kbl,cfl,bxt
5427 * Transition WM are not recommended by HW team for GEN9
5429 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5432 if (INTEL_GEN(dev_priv) >= 11)
5437 /* Display WA #1140: glk,cnl */
5438 if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5441 trans_amount = 10; /* This is configurable amount */
5443 trans_offset_b = trans_min + trans_amount;
5446 * The spec asks for Selected Result Blocks for wm0 (the real value),
5447 * not Result Blocks (the integer value). Pay attention to the capital
5448 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5449 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5450 * and since we later will have to get the ceiling of the sum in the
5451 * transition watermarks calculation, we can just pretend Selected
5452 * Result Blocks is Result Blocks minus 1 and it should work for the
5453 * current platforms.
5455 wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5459 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5460 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5463 res_blocks = wm0_sel_res_b + trans_offset_b;
5467 * Just assume we can enable the transition watermark. After
5468 * computing the DDB we'll come back and disable it if that
5469 * assumption turns out to be false.
5471 wm->trans_wm.plane_res_b = res_blocks + 1;
5472 wm->trans_wm.plane_en = true;
5475 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5476 const struct intel_plane_state *plane_state,
5477 enum plane_id plane_id, int color_plane)
5479 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5480 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5481 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5482 struct skl_wm_params wm_params;
5485 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5486 &wm_params, color_plane);
5490 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5492 if (INTEL_GEN(dev_priv) >= 12)
5493 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5495 skl_compute_transition_wm(crtc_state, &wm_params, wm);
5500 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5501 const struct intel_plane_state *plane_state,
5502 enum plane_id plane_id)
5504 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5505 struct skl_wm_params wm_params;
5508 wm->is_planar = true;
5510 /* uv plane watermarks must also be validated for NV12/Planar */
5511 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5516 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5521 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5522 const struct intel_plane_state *plane_state)
5524 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5525 const struct drm_framebuffer *fb = plane_state->hw.fb;
5526 enum plane_id plane_id = plane->id;
5529 if (!intel_wm_plane_visible(crtc_state, plane_state))
5532 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5537 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5538 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5547 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5548 const struct intel_plane_state *plane_state)
5550 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5551 enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5554 /* Watermarks calculated in master */
5555 if (plane_state->planar_slave)
5558 if (plane_state->planar_linked_plane) {
5559 const struct drm_framebuffer *fb = plane_state->hw.fb;
5560 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5562 drm_WARN_ON(&dev_priv->drm,
5563 !intel_wm_plane_visible(crtc_state, plane_state));
5564 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5565 fb->format->num_planes == 1);
5567 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5572 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5576 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5577 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5586 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5588 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5589 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5590 struct intel_plane *plane;
5591 const struct intel_plane_state *plane_state;
5595 * We'll only calculate watermarks for planes that are actually
5596 * enabled, so make sure all other planes are set as disabled.
5598 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5600 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5603 if (INTEL_GEN(dev_priv) >= 11)
5604 ret = icl_build_plane_wm(crtc_state, plane_state);
5606 ret = skl_build_plane_wm(crtc_state, plane_state);
5614 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5616 const struct skl_ddb_entry *entry)
5619 intel_de_write_fw(dev_priv, reg,
5620 (entry->end - 1) << 16 | entry->start);
5622 intel_de_write_fw(dev_priv, reg, 0);
5625 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5627 const struct skl_wm_level *level)
5631 if (level->plane_en)
5633 if (level->ignore_lines)
5634 val |= PLANE_WM_IGNORE_LINES;
5635 val |= level->plane_res_b;
5636 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5638 intel_de_write_fw(dev_priv, reg, val);
5641 void skl_write_plane_wm(struct intel_plane *plane,
5642 const struct intel_crtc_state *crtc_state)
5644 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5645 int level, max_level = ilk_wm_max_level(dev_priv);
5646 enum plane_id plane_id = plane->id;
5647 enum pipe pipe = plane->pipe;
5648 const struct skl_plane_wm *wm =
5649 &crtc_state->wm.skl.optimal.planes[plane_id];
5650 const struct skl_ddb_entry *ddb_y =
5651 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5652 const struct skl_ddb_entry *ddb_uv =
5653 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5655 for (level = 0; level <= max_level; level++) {
5656 const struct skl_wm_level *wm_level;
5658 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5660 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5663 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5666 if (INTEL_GEN(dev_priv) >= 11) {
5667 skl_ddb_entry_write(dev_priv,
5668 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5673 swap(ddb_y, ddb_uv);
5675 skl_ddb_entry_write(dev_priv,
5676 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5677 skl_ddb_entry_write(dev_priv,
5678 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5681 void skl_write_cursor_wm(struct intel_plane *plane,
5682 const struct intel_crtc_state *crtc_state)
5684 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5685 int level, max_level = ilk_wm_max_level(dev_priv);
5686 enum plane_id plane_id = plane->id;
5687 enum pipe pipe = plane->pipe;
5688 const struct skl_plane_wm *wm =
5689 &crtc_state->wm.skl.optimal.planes[plane_id];
5690 const struct skl_ddb_entry *ddb =
5691 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5693 for (level = 0; level <= max_level; level++) {
5694 const struct skl_wm_level *wm_level;
5696 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5698 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5701 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5703 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5706 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5707 const struct skl_wm_level *l2)
5709 return l1->plane_en == l2->plane_en &&
5710 l1->ignore_lines == l2->ignore_lines &&
5711 l1->plane_res_l == l2->plane_res_l &&
5712 l1->plane_res_b == l2->plane_res_b;
5715 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5716 const struct skl_plane_wm *wm1,
5717 const struct skl_plane_wm *wm2)
5719 int level, max_level = ilk_wm_max_level(dev_priv);
5721 for (level = 0; level <= max_level; level++) {
5723 * We don't check uv_wm as the hardware doesn't actually
5724 * use it. It only gets used for calculating the required
5727 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5731 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5734 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5735 const struct skl_ddb_entry *b)
5737 return a->start < b->end && b->start < a->end;
5740 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5741 const struct skl_ddb_entry *entries,
5742 int num_entries, int ignore_idx)
5746 for (i = 0; i < num_entries; i++) {
5747 if (i != ignore_idx &&
5748 skl_ddb_entries_overlap(ddb, &entries[i]))
5756 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5757 struct intel_crtc_state *new_crtc_state)
5759 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5760 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5761 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5762 struct intel_plane *plane;
5764 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5765 struct intel_plane_state *plane_state;
5766 enum plane_id plane_id = plane->id;
5768 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5769 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5770 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5771 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5774 plane_state = intel_atomic_get_plane_state(state, plane);
5775 if (IS_ERR(plane_state))
5776 return PTR_ERR(plane_state);
5778 new_crtc_state->update_planes |= BIT(plane_id);
5785 skl_compute_ddb(struct intel_atomic_state *state)
5787 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5788 const struct intel_dbuf_state *old_dbuf_state;
5789 const struct intel_dbuf_state *new_dbuf_state;
5790 const struct intel_crtc_state *old_crtc_state;
5791 struct intel_crtc_state *new_crtc_state;
5792 struct intel_crtc *crtc;
5795 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5796 new_crtc_state, i) {
5797 ret = skl_allocate_pipe_ddb(new_crtc_state);
5801 ret = skl_ddb_add_affected_planes(old_crtc_state,
5807 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5808 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
5810 if (new_dbuf_state &&
5811 new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices)
5812 drm_dbg_kms(&dev_priv->drm,
5813 "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5814 old_dbuf_state->enabled_slices,
5815 new_dbuf_state->enabled_slices,
5816 INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5821 static char enast(bool enable)
5823 return enable ? '*' : ' ';
5827 skl_print_wm_changes(struct intel_atomic_state *state)
5829 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5830 const struct intel_crtc_state *old_crtc_state;
5831 const struct intel_crtc_state *new_crtc_state;
5832 struct intel_plane *plane;
5833 struct intel_crtc *crtc;
5836 if (!drm_debug_enabled(DRM_UT_KMS))
5839 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5840 new_crtc_state, i) {
5841 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5843 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5844 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5846 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5847 enum plane_id plane_id = plane->id;
5848 const struct skl_ddb_entry *old, *new;
5850 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5851 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5853 if (skl_ddb_entry_equal(old, new))
5856 drm_dbg_kms(&dev_priv->drm,
5857 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5858 plane->base.base.id, plane->base.name,
5859 old->start, old->end, new->start, new->end,
5860 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5863 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5864 enum plane_id plane_id = plane->id;
5865 const struct skl_plane_wm *old_wm, *new_wm;
5867 old_wm = &old_pipe_wm->planes[plane_id];
5868 new_wm = &new_pipe_wm->planes[plane_id];
5870 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5873 drm_dbg_kms(&dev_priv->drm,
5874 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5875 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5876 plane->base.base.id, plane->base.name,
5877 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5878 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5879 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5880 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5881 enast(old_wm->trans_wm.plane_en),
5882 enast(old_wm->sagv_wm0.plane_en),
5883 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5884 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5885 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5886 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5887 enast(new_wm->trans_wm.plane_en),
5888 enast(new_wm->sagv_wm0.plane_en));
5890 drm_dbg_kms(&dev_priv->drm,
5891 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5892 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5893 plane->base.base.id, plane->base.name,
5894 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5895 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5896 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5897 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5898 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5899 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5900 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5901 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5902 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5903 enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5905 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5906 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5907 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5908 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5909 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5910 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5911 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5912 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5913 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5914 enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5916 drm_dbg_kms(&dev_priv->drm,
5917 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5918 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5919 plane->base.base.id, plane->base.name,
5920 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5921 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5922 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5923 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5924 old_wm->trans_wm.plane_res_b,
5925 old_wm->sagv_wm0.plane_res_b,
5926 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5927 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5928 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5929 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5930 new_wm->trans_wm.plane_res_b,
5931 new_wm->sagv_wm0.plane_res_b);
5933 drm_dbg_kms(&dev_priv->drm,
5934 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5935 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5936 plane->base.base.id, plane->base.name,
5937 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5938 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5939 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5940 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5941 old_wm->trans_wm.min_ddb_alloc,
5942 old_wm->sagv_wm0.min_ddb_alloc,
5943 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5944 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5945 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5946 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5947 new_wm->trans_wm.min_ddb_alloc,
5948 new_wm->sagv_wm0.min_ddb_alloc);
5953 static int intel_add_affected_pipes(struct intel_atomic_state *state,
5956 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5957 struct intel_crtc *crtc;
5959 for_each_intel_crtc(&dev_priv->drm, crtc) {
5960 struct intel_crtc_state *crtc_state;
5962 if ((pipe_mask & BIT(crtc->pipe)) == 0)
5965 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5966 if (IS_ERR(crtc_state))
5967 return PTR_ERR(crtc_state);
5974 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5976 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5977 struct intel_crtc_state *crtc_state;
5978 struct intel_crtc *crtc;
5981 if (dev_priv->wm.distrust_bios_wm) {
5983 * skl_ddb_get_pipe_allocation_limits() currently requires
5984 * all active pipes to be included in the state so that
5985 * it can redistribute the dbuf among them, and it really
5986 * wants to recompute things when distrust_bios_wm is set
5987 * so we add all the pipes to the state.
5989 ret = intel_add_affected_pipes(state, ~0);
5994 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
5995 struct intel_dbuf_state *new_dbuf_state;
5996 const struct intel_dbuf_state *old_dbuf_state;
5998 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5999 if (IS_ERR(new_dbuf_state))
6000 return PTR_ERR(new_dbuf_state);
6002 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6004 new_dbuf_state->active_pipes =
6005 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6007 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes)
6010 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6015 * skl_ddb_get_pipe_allocation_limits() currently requires
6016 * all active pipes to be included in the state so that
6017 * it can redistribute the dbuf among them.
6019 ret = intel_add_affected_pipes(state,
6020 new_dbuf_state->active_pipes);
6031 * To make sure the cursor watermark registers are always consistent
6032 * with our computed state the following scenario needs special
6036 * 2. move cursor entirely offscreen
6039 * Step 2. does call .disable_plane() but does not zero the watermarks
6040 * (since we consider an offscreen cursor still active for the purposes
6041 * of watermarks). Step 3. would not normally call .disable_plane()
6042 * because the actual plane visibility isn't changing, and we don't
6043 * deallocate the cursor ddb until the pipe gets disabled. So we must
6044 * force step 3. to call .disable_plane() to update the watermark
6045 * registers properly.
6047 * Other planes do not suffer from this issues as their watermarks are
6048 * calculated based on the actual plane visibility. The only time this
6049 * can trigger for the other planes is during the initial readout as the
6050 * default value of the watermarks registers is not zero.
6052 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6053 struct intel_crtc *crtc)
6055 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6056 const struct intel_crtc_state *old_crtc_state =
6057 intel_atomic_get_old_crtc_state(state, crtc);
6058 struct intel_crtc_state *new_crtc_state =
6059 intel_atomic_get_new_crtc_state(state, crtc);
6060 struct intel_plane *plane;
6062 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6063 struct intel_plane_state *plane_state;
6064 enum plane_id plane_id = plane->id;
6067 * Force a full wm update for every plane on modeset.
6068 * Required because the reset value of the wm registers
6069 * is non-zero, whereas we want all disabled planes to
6070 * have zero watermarks. So if we turn off the relevant
6071 * power well the hardware state will go out of sync
6072 * with the software state.
6074 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6075 skl_plane_wm_equals(dev_priv,
6076 &old_crtc_state->wm.skl.optimal.planes[plane_id],
6077 &new_crtc_state->wm.skl.optimal.planes[plane_id]))
6080 plane_state = intel_atomic_get_plane_state(state, plane);
6081 if (IS_ERR(plane_state))
6082 return PTR_ERR(plane_state);
6084 new_crtc_state->update_planes |= BIT(plane_id);
6091 skl_compute_wm(struct intel_atomic_state *state)
6093 struct intel_crtc *crtc;
6094 struct intel_crtc_state *new_crtc_state;
6095 struct intel_crtc_state *old_crtc_state;
6098 ret = skl_ddb_add_affected_pipes(state);
6103 * Calculate WM's for all pipes that are part of this transaction.
6104 * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6105 * weren't otherwise being modified if pipe allocations had to change.
6107 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6108 new_crtc_state, i) {
6109 ret = skl_build_pipe_wm(new_crtc_state);
6114 ret = skl_compute_ddb(state);
6118 ret = intel_compute_sagv_mask(state);
6123 * skl_compute_ddb() will have adjusted the final watermarks
6124 * based on how much ddb is available. Now we can actually
6125 * check if the final watermarks changed.
6127 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6128 new_crtc_state, i) {
6129 ret = skl_wm_add_affected_planes(state, crtc);
6134 skl_print_wm_changes(state);
6139 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6140 struct intel_wm_config *config)
6142 struct intel_crtc *crtc;
6144 /* Compute the currently _active_ config */
6145 for_each_intel_crtc(&dev_priv->drm, crtc) {
6146 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6148 if (!wm->pipe_enabled)
6151 config->sprites_enabled |= wm->sprites_enabled;
6152 config->sprites_scaled |= wm->sprites_scaled;
6153 config->num_pipes_active++;
6157 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6159 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6160 struct ilk_wm_maximums max;
6161 struct intel_wm_config config = {};
6162 struct ilk_wm_values results = {};
6163 enum intel_ddb_partitioning partitioning;
6165 ilk_compute_wm_config(dev_priv, &config);
6167 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6168 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6170 /* 5/6 split only in single pipe config on IVB+ */
6171 if (INTEL_GEN(dev_priv) >= 7 &&
6172 config.num_pipes_active == 1 && config.sprites_enabled) {
6173 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6174 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6176 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6178 best_lp_wm = &lp_wm_1_2;
6181 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6182 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6184 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6186 ilk_write_wm_values(dev_priv, &results);
6189 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6190 struct intel_crtc *crtc)
6192 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6193 const struct intel_crtc_state *crtc_state =
6194 intel_atomic_get_new_crtc_state(state, crtc);
6196 mutex_lock(&dev_priv->wm.wm_mutex);
6197 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6198 ilk_program_watermarks(dev_priv);
6199 mutex_unlock(&dev_priv->wm.wm_mutex);
6202 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6203 struct intel_crtc *crtc)
6205 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6206 const struct intel_crtc_state *crtc_state =
6207 intel_atomic_get_new_crtc_state(state, crtc);
6209 if (!crtc_state->wm.need_postvbl_update)
6212 mutex_lock(&dev_priv->wm.wm_mutex);
6213 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6214 ilk_program_watermarks(dev_priv);
6215 mutex_unlock(&dev_priv->wm.wm_mutex);
6218 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6220 level->plane_en = val & PLANE_WM_EN;
6221 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6222 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6223 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6224 PLANE_WM_LINES_MASK;
6227 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6228 struct skl_pipe_wm *out)
6230 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6231 enum pipe pipe = crtc->pipe;
6232 int level, max_level;
6233 enum plane_id plane_id;
6236 max_level = ilk_wm_max_level(dev_priv);
6238 for_each_plane_id_on_crtc(crtc, plane_id) {
6239 struct skl_plane_wm *wm = &out->planes[plane_id];
6241 for (level = 0; level <= max_level; level++) {
6242 if (plane_id != PLANE_CURSOR)
6243 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6245 val = I915_READ(CUR_WM(pipe, level));
6247 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6250 if (INTEL_GEN(dev_priv) >= 12)
6251 wm->sagv_wm0 = wm->wm[0];
6253 if (plane_id != PLANE_CURSOR)
6254 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6256 val = I915_READ(CUR_WM_TRANS(pipe));
6258 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6265 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6267 struct intel_crtc *crtc;
6268 struct intel_crtc_state *crtc_state;
6270 for_each_intel_crtc(&dev_priv->drm, crtc) {
6271 crtc_state = to_intel_crtc_state(crtc->base.state);
6273 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6276 if (dev_priv->active_pipes) {
6277 /* Fully recompute DDB on first atomic commit */
6278 dev_priv->wm.distrust_bios_wm = true;
6282 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6284 struct drm_device *dev = crtc->base.dev;
6285 struct drm_i915_private *dev_priv = to_i915(dev);
6286 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6287 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6288 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6289 enum pipe pipe = crtc->pipe;
6290 static const i915_reg_t wm0_pipe_reg[] = {
6291 [PIPE_A] = WM0_PIPEA_ILK,
6292 [PIPE_B] = WM0_PIPEB_ILK,
6293 [PIPE_C] = WM0_PIPEC_IVB,
6296 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6298 memset(active, 0, sizeof(*active));
6300 active->pipe_enabled = crtc->active;
6302 if (active->pipe_enabled) {
6303 u32 tmp = hw->wm_pipe[pipe];
6306 * For active pipes LP0 watermark is marked as
6307 * enabled, and LP1+ watermaks as disabled since
6308 * we can't really reverse compute them in case
6309 * multiple pipes are active.
6311 active->wm[0].enable = true;
6312 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6313 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6314 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6316 int level, max_level = ilk_wm_max_level(dev_priv);
6319 * For inactive pipes, all watermark levels
6320 * should be marked as enabled but zeroed,
6321 * which is what we'd compute them to.
6323 for (level = 0; level <= max_level; level++)
6324 active->wm[level].enable = true;
6327 crtc->wm.active.ilk = *active;
6330 #define _FW_WM(value, plane) \
6331 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6332 #define _FW_WM_VLV(value, plane) \
6333 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6335 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6336 struct g4x_wm_values *wm)
6340 tmp = I915_READ(DSPFW1);
6341 wm->sr.plane = _FW_WM(tmp, SR);
6342 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6343 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6344 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6346 tmp = I915_READ(DSPFW2);
6347 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6348 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6349 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6350 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6351 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6352 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6354 tmp = I915_READ(DSPFW3);
6355 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6356 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6357 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6358 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6361 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6362 struct vlv_wm_values *wm)
6367 for_each_pipe(dev_priv, pipe) {
6368 tmp = I915_READ(VLV_DDL(pipe));
6370 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6371 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6372 wm->ddl[pipe].plane[PLANE_CURSOR] =
6373 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6374 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6375 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6376 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6377 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6380 tmp = I915_READ(DSPFW1);
6381 wm->sr.plane = _FW_WM(tmp, SR);
6382 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6383 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6384 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6386 tmp = I915_READ(DSPFW2);
6387 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6388 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6389 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6391 tmp = I915_READ(DSPFW3);
6392 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6394 if (IS_CHERRYVIEW(dev_priv)) {
6395 tmp = I915_READ(DSPFW7_CHV);
6396 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6397 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6399 tmp = I915_READ(DSPFW8_CHV);
6400 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6401 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6403 tmp = I915_READ(DSPFW9_CHV);
6404 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6405 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6407 tmp = I915_READ(DSPHOWM);
6408 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6409 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6410 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6411 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6412 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6413 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6414 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6415 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6416 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6417 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6419 tmp = I915_READ(DSPFW7);
6420 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6421 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6423 tmp = I915_READ(DSPHOWM);
6424 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6425 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6426 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6427 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6428 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6429 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6430 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6437 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6439 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6440 struct intel_crtc *crtc;
6442 g4x_read_wm_values(dev_priv, wm);
6444 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6446 for_each_intel_crtc(&dev_priv->drm, crtc) {
6447 struct intel_crtc_state *crtc_state =
6448 to_intel_crtc_state(crtc->base.state);
6449 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6450 struct g4x_pipe_wm *raw;
6451 enum pipe pipe = crtc->pipe;
6452 enum plane_id plane_id;
6453 int level, max_level;
6455 active->cxsr = wm->cxsr;
6456 active->hpll_en = wm->hpll_en;
6457 active->fbc_en = wm->fbc_en;
6459 active->sr = wm->sr;
6460 active->hpll = wm->hpll;
6462 for_each_plane_id_on_crtc(crtc, plane_id) {
6463 active->wm.plane[plane_id] =
6464 wm->pipe[pipe].plane[plane_id];
6467 if (wm->cxsr && wm->hpll_en)
6468 max_level = G4X_WM_LEVEL_HPLL;
6470 max_level = G4X_WM_LEVEL_SR;
6472 max_level = G4X_WM_LEVEL_NORMAL;
6474 level = G4X_WM_LEVEL_NORMAL;
6475 raw = &crtc_state->wm.g4x.raw[level];
6476 for_each_plane_id_on_crtc(crtc, plane_id)
6477 raw->plane[plane_id] = active->wm.plane[plane_id];
6479 if (++level > max_level)
6482 raw = &crtc_state->wm.g4x.raw[level];
6483 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6484 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6485 raw->plane[PLANE_SPRITE0] = 0;
6486 raw->fbc = active->sr.fbc;
6488 if (++level > max_level)
6491 raw = &crtc_state->wm.g4x.raw[level];
6492 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6493 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6494 raw->plane[PLANE_SPRITE0] = 0;
6495 raw->fbc = active->hpll.fbc;
6498 for_each_plane_id_on_crtc(crtc, plane_id)
6499 g4x_raw_plane_wm_set(crtc_state, level,
6500 plane_id, USHRT_MAX);
6501 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6503 crtc_state->wm.g4x.optimal = *active;
6504 crtc_state->wm.g4x.intermediate = *active;
6506 drm_dbg_kms(&dev_priv->drm,
6507 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6509 wm->pipe[pipe].plane[PLANE_PRIMARY],
6510 wm->pipe[pipe].plane[PLANE_CURSOR],
6511 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6514 drm_dbg_kms(&dev_priv->drm,
6515 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6516 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6517 drm_dbg_kms(&dev_priv->drm,
6518 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6519 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6520 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6521 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6524 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6526 struct intel_plane *plane;
6527 struct intel_crtc *crtc;
6529 mutex_lock(&dev_priv->wm.wm_mutex);
6531 for_each_intel_plane(&dev_priv->drm, plane) {
6532 struct intel_crtc *crtc =
6533 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6534 struct intel_crtc_state *crtc_state =
6535 to_intel_crtc_state(crtc->base.state);
6536 struct intel_plane_state *plane_state =
6537 to_intel_plane_state(plane->base.state);
6538 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6539 enum plane_id plane_id = plane->id;
6542 if (plane_state->uapi.visible)
6545 for (level = 0; level < 3; level++) {
6546 struct g4x_pipe_wm *raw =
6547 &crtc_state->wm.g4x.raw[level];
6549 raw->plane[plane_id] = 0;
6550 wm_state->wm.plane[plane_id] = 0;
6553 if (plane_id == PLANE_PRIMARY) {
6554 for (level = 0; level < 3; level++) {
6555 struct g4x_pipe_wm *raw =
6556 &crtc_state->wm.g4x.raw[level];
6560 wm_state->sr.fbc = 0;
6561 wm_state->hpll.fbc = 0;
6562 wm_state->fbc_en = false;
6566 for_each_intel_crtc(&dev_priv->drm, crtc) {
6567 struct intel_crtc_state *crtc_state =
6568 to_intel_crtc_state(crtc->base.state);
6570 crtc_state->wm.g4x.intermediate =
6571 crtc_state->wm.g4x.optimal;
6572 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6575 g4x_program_watermarks(dev_priv);
6577 mutex_unlock(&dev_priv->wm.wm_mutex);
6580 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6582 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6583 struct intel_crtc *crtc;
6586 vlv_read_wm_values(dev_priv, wm);
6588 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6589 wm->level = VLV_WM_LEVEL_PM2;
6591 if (IS_CHERRYVIEW(dev_priv)) {
6592 vlv_punit_get(dev_priv);
6594 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6595 if (val & DSP_MAXFIFO_PM5_ENABLE)
6596 wm->level = VLV_WM_LEVEL_PM5;
6599 * If DDR DVFS is disabled in the BIOS, Punit
6600 * will never ack the request. So if that happens
6601 * assume we don't have to enable/disable DDR DVFS
6602 * dynamically. To test that just set the REQ_ACK
6603 * bit to poke the Punit, but don't change the
6604 * HIGH/LOW bits so that we don't actually change
6605 * the current state.
6607 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6608 val |= FORCE_DDR_FREQ_REQ_ACK;
6609 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6611 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6612 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6613 drm_dbg_kms(&dev_priv->drm,
6614 "Punit not acking DDR DVFS request, "
6615 "assuming DDR DVFS is disabled\n");
6616 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6618 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6619 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6620 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6623 vlv_punit_put(dev_priv);
6626 for_each_intel_crtc(&dev_priv->drm, crtc) {
6627 struct intel_crtc_state *crtc_state =
6628 to_intel_crtc_state(crtc->base.state);
6629 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6630 const struct vlv_fifo_state *fifo_state =
6631 &crtc_state->wm.vlv.fifo_state;
6632 enum pipe pipe = crtc->pipe;
6633 enum plane_id plane_id;
6636 vlv_get_fifo_size(crtc_state);
6638 active->num_levels = wm->level + 1;
6639 active->cxsr = wm->cxsr;
6641 for (level = 0; level < active->num_levels; level++) {
6642 struct g4x_pipe_wm *raw =
6643 &crtc_state->wm.vlv.raw[level];
6645 active->sr[level].plane = wm->sr.plane;
6646 active->sr[level].cursor = wm->sr.cursor;
6648 for_each_plane_id_on_crtc(crtc, plane_id) {
6649 active->wm[level].plane[plane_id] =
6650 wm->pipe[pipe].plane[plane_id];
6652 raw->plane[plane_id] =
6653 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6654 fifo_state->plane[plane_id]);
6658 for_each_plane_id_on_crtc(crtc, plane_id)
6659 vlv_raw_plane_wm_set(crtc_state, level,
6660 plane_id, USHRT_MAX);
6661 vlv_invalidate_wms(crtc, active, level);
6663 crtc_state->wm.vlv.optimal = *active;
6664 crtc_state->wm.vlv.intermediate = *active;
6666 drm_dbg_kms(&dev_priv->drm,
6667 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6669 wm->pipe[pipe].plane[PLANE_PRIMARY],
6670 wm->pipe[pipe].plane[PLANE_CURSOR],
6671 wm->pipe[pipe].plane[PLANE_SPRITE0],
6672 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6675 drm_dbg_kms(&dev_priv->drm,
6676 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6677 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6680 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6682 struct intel_plane *plane;
6683 struct intel_crtc *crtc;
6685 mutex_lock(&dev_priv->wm.wm_mutex);
6687 for_each_intel_plane(&dev_priv->drm, plane) {
6688 struct intel_crtc *crtc =
6689 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6690 struct intel_crtc_state *crtc_state =
6691 to_intel_crtc_state(crtc->base.state);
6692 struct intel_plane_state *plane_state =
6693 to_intel_plane_state(plane->base.state);
6694 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6695 const struct vlv_fifo_state *fifo_state =
6696 &crtc_state->wm.vlv.fifo_state;
6697 enum plane_id plane_id = plane->id;
6700 if (plane_state->uapi.visible)
6703 for (level = 0; level < wm_state->num_levels; level++) {
6704 struct g4x_pipe_wm *raw =
6705 &crtc_state->wm.vlv.raw[level];
6707 raw->plane[plane_id] = 0;
6709 wm_state->wm[level].plane[plane_id] =
6710 vlv_invert_wm_value(raw->plane[plane_id],
6711 fifo_state->plane[plane_id]);
6715 for_each_intel_crtc(&dev_priv->drm, crtc) {
6716 struct intel_crtc_state *crtc_state =
6717 to_intel_crtc_state(crtc->base.state);
6719 crtc_state->wm.vlv.intermediate =
6720 crtc_state->wm.vlv.optimal;
6721 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6724 vlv_program_watermarks(dev_priv);
6726 mutex_unlock(&dev_priv->wm.wm_mutex);
6730 * FIXME should probably kill this and improve
6731 * the real watermark readout/sanitation instead
6733 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6735 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6736 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6737 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6740 * Don't touch WM1S_LP_EN here.
6741 * Doing so could cause underruns.
6745 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6747 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6748 struct intel_crtc *crtc;
6750 ilk_init_lp_watermarks(dev_priv);
6752 for_each_intel_crtc(&dev_priv->drm, crtc)
6753 ilk_pipe_wm_get_hw_state(crtc);
6755 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6756 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6757 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6759 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6760 if (INTEL_GEN(dev_priv) >= 7) {
6761 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6762 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6765 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6766 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6767 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6768 else if (IS_IVYBRIDGE(dev_priv))
6769 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6770 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6773 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6777 * intel_update_watermarks - update FIFO watermark values based on current modes
6778 * @crtc: the #intel_crtc on which to compute the WM
6780 * Calculate watermark values for the various WM regs based on current mode
6781 * and plane configuration.
6783 * There are several cases to deal with here:
6784 * - normal (i.e. non-self-refresh)
6785 * - self-refresh (SR) mode
6786 * - lines are large relative to FIFO size (buffer can hold up to 2)
6787 * - lines are small relative to FIFO size (buffer can hold more than 2
6788 * lines), so need to account for TLB latency
6790 * The normal calculation is:
6791 * watermark = dotclock * bytes per pixel * latency
6792 * where latency is platform & configuration dependent (we assume pessimal
6795 * The SR calculation is:
6796 * watermark = (trunc(latency/line time)+1) * surface width *
6799 * line time = htotal / dotclock
6800 * surface width = hdisplay for normal plane and 64 for cursor
6801 * and latency is assumed to be high, as above.
6803 * The final value programmed to the register should always be rounded up,
6804 * and include an extra 2 entries to account for clock crossings.
6806 * We don't use the sprite, so we can ignore that. And on Crestline we have
6807 * to set the non-SR watermarks to 8.
6809 void intel_update_watermarks(struct intel_crtc *crtc)
6811 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6813 if (dev_priv->display.update_wm)
6814 dev_priv->display.update_wm(crtc);
6817 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6821 if (!HAS_IPC(dev_priv))
6824 val = I915_READ(DISP_ARB_CTL2);
6826 if (dev_priv->ipc_enabled)
6827 val |= DISP_IPC_ENABLE;
6829 val &= ~DISP_IPC_ENABLE;
6831 I915_WRITE(DISP_ARB_CTL2, val);
6834 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6836 /* Display WA #0477 WaDisableIPC: skl */
6837 if (IS_SKYLAKE(dev_priv))
6840 /* Display WA #1141: SKL:all KBL:all CFL */
6841 if (IS_KABYLAKE(dev_priv) ||
6842 IS_COFFEELAKE(dev_priv) ||
6843 IS_COMETLAKE(dev_priv))
6844 return dev_priv->dram_info.symmetric_memory;
6849 void intel_init_ipc(struct drm_i915_private *dev_priv)
6851 if (!HAS_IPC(dev_priv))
6854 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6856 intel_enable_ipc(dev_priv);
6859 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6862 * On Ibex Peak and Cougar Point, we need to disable clock
6863 * gating for the panel power sequencer or it will fail to
6864 * start up when no ports are active.
6866 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6869 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6873 for_each_pipe(dev_priv, pipe) {
6874 I915_WRITE(DSPCNTR(pipe),
6875 I915_READ(DSPCNTR(pipe)) |
6876 DISPPLANE_TRICKLE_FEED_DISABLE);
6878 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6879 POSTING_READ(DSPSURF(pipe));
6883 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6885 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6889 * WaFbcDisableDpfcClockGating:ilk
6891 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6892 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6893 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6895 I915_WRITE(PCH_3DCGDIS0,
6896 MARIUNIT_CLOCK_GATE_DISABLE |
6897 SVSMUNIT_CLOCK_GATE_DISABLE);
6898 I915_WRITE(PCH_3DCGDIS1,
6899 VFMUNIT_CLOCK_GATE_DISABLE);
6902 * According to the spec the following bits should be set in
6903 * order to enable memory self-refresh
6904 * The bit 22/21 of 0x42004
6905 * The bit 5 of 0x42020
6906 * The bit 15 of 0x45000
6908 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6909 (I915_READ(ILK_DISPLAY_CHICKEN2) |
6910 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6911 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6912 I915_WRITE(DISP_ARB_CTL,
6913 (I915_READ(DISP_ARB_CTL) |
6917 * Based on the document from hardware guys the following bits
6918 * should be set unconditionally in order to enable FBC.
6919 * The bit 22 of 0x42000
6920 * The bit 22 of 0x42004
6921 * The bit 7,8,9 of 0x42020.
6923 if (IS_IRONLAKE_M(dev_priv)) {
6924 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6925 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6926 I915_READ(ILK_DISPLAY_CHICKEN1) |
6928 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6929 I915_READ(ILK_DISPLAY_CHICKEN2) |
6933 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6935 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6936 I915_READ(ILK_DISPLAY_CHICKEN2) |
6937 ILK_ELPIN_409_SELECT);
6939 g4x_disable_trickle_feed(dev_priv);
6941 ibx_init_clock_gating(dev_priv);
6944 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6950 * On Ibex Peak and Cougar Point, we need to disable clock
6951 * gating for the panel power sequencer or it will fail to
6952 * start up when no ports are active.
6954 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6955 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6956 PCH_CPUNIT_CLOCK_GATE_DISABLE);
6957 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6958 DPLS_EDP_PPS_FIX_DIS);
6959 /* The below fixes the weird display corruption, a few pixels shifted
6960 * downward, on (only) LVDS of some HP laptops with IVY.
6962 for_each_pipe(dev_priv, pipe) {
6963 val = I915_READ(TRANS_CHICKEN2(pipe));
6964 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6965 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6966 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6967 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6968 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6969 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6970 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6972 /* WADP0ClockGatingDisable */
6973 for_each_pipe(dev_priv, pipe) {
6974 I915_WRITE(TRANS_CHICKEN1(pipe),
6975 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6979 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6983 tmp = I915_READ(MCH_SSKPD);
6984 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6985 drm_dbg_kms(&dev_priv->drm,
6986 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6990 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6992 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6994 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6996 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6997 I915_READ(ILK_DISPLAY_CHICKEN2) |
6998 ILK_ELPIN_409_SELECT);
7000 I915_WRITE(GEN6_UCGCTL1,
7001 I915_READ(GEN6_UCGCTL1) |
7002 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7003 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7005 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7006 * gating disable must be set. Failure to set it results in
7007 * flickering pixels due to Z write ordering failures after
7008 * some amount of runtime in the Mesa "fire" demo, and Unigine
7009 * Sanctuary and Tropics, and apparently anything else with
7010 * alpha test or pixel discard.
7012 * According to the spec, bit 11 (RCCUNIT) must also be set,
7013 * but we didn't debug actual testcases to find it out.
7015 * WaDisableRCCUnitClockGating:snb
7016 * WaDisableRCPBUnitClockGating:snb
7018 I915_WRITE(GEN6_UCGCTL2,
7019 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7020 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7023 * According to the spec the following bits should be
7024 * set in order to enable memory self-refresh and fbc:
7025 * The bit21 and bit22 of 0x42000
7026 * The bit21 and bit22 of 0x42004
7027 * The bit5 and bit7 of 0x42020
7028 * The bit14 of 0x70180
7029 * The bit14 of 0x71180
7031 * WaFbcAsynchFlipDisableFbcQueue:snb
7033 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7034 I915_READ(ILK_DISPLAY_CHICKEN1) |
7035 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7036 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7037 I915_READ(ILK_DISPLAY_CHICKEN2) |
7038 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7039 I915_WRITE(ILK_DSPCLK_GATE_D,
7040 I915_READ(ILK_DSPCLK_GATE_D) |
7041 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7042 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7044 g4x_disable_trickle_feed(dev_priv);
7046 cpt_init_clock_gating(dev_priv);
7048 gen6_check_mch_setup(dev_priv);
7051 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7054 * TODO: this bit should only be enabled when really needed, then
7055 * disabled when not needed anymore in order to save power.
7057 if (HAS_PCH_LPT_LP(dev_priv))
7058 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7059 I915_READ(SOUTH_DSPCLK_GATE_D) |
7060 PCH_LP_PARTITION_LEVEL_DISABLE);
7062 /* WADPOClockGatingDisable:hsw */
7063 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7064 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7065 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7068 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7070 if (HAS_PCH_LPT_LP(dev_priv)) {
7071 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7073 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7074 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7078 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7079 int general_prio_credits,
7080 int high_prio_credits)
7085 /* WaTempDisableDOPClkGating:bdw */
7086 misccpctl = I915_READ(GEN7_MISCCPCTL);
7087 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7089 val = I915_READ(GEN8_L3SQCREG1);
7090 val &= ~L3_PRIO_CREDITS_MASK;
7091 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7092 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7093 I915_WRITE(GEN8_L3SQCREG1, val);
7096 * Wait at least 100 clocks before re-enabling clock gating.
7097 * See the definition of L3SQCREG1 in BSpec.
7099 POSTING_READ(GEN8_L3SQCREG1);
7101 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7104 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7106 /* Wa_1409120013:icl,ehl */
7107 I915_WRITE(ILK_DPFC_CHICKEN,
7108 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7110 /* This is not an Wa. Enable to reduce Sampler power */
7111 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7112 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7114 /*Wa_14010594013:icl, ehl */
7115 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7116 0, CNL_DELAY_PMRSP);
7119 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7121 u32 vd_pg_enable = 0;
7124 /* Wa_1409120013:tgl */
7125 I915_WRITE(ILK_DPFC_CHICKEN,
7126 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7128 /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7129 for (i = 0; i < I915_MAX_VCS; i++) {
7130 if (HAS_ENGINE(&dev_priv->gt, _VCS(i)))
7131 vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7132 VDN_MFX_POWERGATE_ENABLE(i);
7135 I915_WRITE(POWERGATE_ENABLE,
7136 I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7138 /* Wa_1409825376:tgl (pre-prod)*/
7139 if (IS_TGL_DISP_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_B1))
7140 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7141 TGL_VRH_GATING_DIS);
7143 /* Wa_14011059788:tgl */
7144 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7148 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7150 if (!HAS_PCH_CNP(dev_priv))
7153 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7154 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7155 CNP_PWM_CGE_GATING_DISABLE);
7158 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7161 cnp_init_clock_gating(dev_priv);
7163 /* This is not an Wa. Enable for better image quality */
7164 I915_WRITE(_3D_CHICKEN3,
7165 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7167 /* WaEnableChickenDCPR:cnl */
7168 I915_WRITE(GEN8_CHICKEN_DCPR_1,
7169 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7172 * WaFbcWakeMemOn:cnl
7173 * Display WA #0859: cnl
7175 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7176 DISP_FBC_MEMORY_WAKE);
7178 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7179 /* ReadHitWriteOnlyDisable:cnl */
7180 val |= RCCUNIT_CLKGATE_DIS;
7181 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7183 /* Wa_2201832410:cnl */
7184 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7185 val |= GWUNIT_CLKGATE_DIS;
7186 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7188 /* WaDisableVFclkgate:cnl */
7189 /* WaVFUnitClockGatingDisable:cnl */
7190 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7191 val |= VFUNIT_CLKGATE_DIS;
7192 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7195 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7197 cnp_init_clock_gating(dev_priv);
7198 gen9_init_clock_gating(dev_priv);
7201 * WaFbcTurnOffFbcWatermark:cfl
7202 * Display WA #0562: cfl
7204 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7208 * WaFbcNukeOnHostModify:cfl
7209 * Display WA #0873: cfl
7211 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7212 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7215 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7217 gen9_init_clock_gating(dev_priv);
7219 /* WaDisableSDEUnitClockGating:kbl */
7220 if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0))
7221 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7222 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7224 /* WaDisableGamClockGating:kbl */
7225 if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0))
7226 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7227 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7230 * WaFbcTurnOffFbcWatermark:kbl
7231 * Display WA #0562: kbl
7233 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7237 * WaFbcNukeOnHostModify:kbl
7238 * Display WA #0873: kbl
7240 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7241 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7244 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7246 gen9_init_clock_gating(dev_priv);
7248 /* WaDisableDopClockGating:skl */
7249 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL) &
7250 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7252 /* WAC6entrylatency:skl */
7253 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7254 FBC_LLC_FULLY_OPEN);
7257 * WaFbcTurnOffFbcWatermark:skl
7258 * Display WA #0562: skl
7260 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7264 * WaFbcNukeOnHostModify:skl
7265 * Display WA #0873: skl
7267 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7268 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7271 * WaFbcHighMemBwCorruptionAvoidance:skl
7272 * Display WA #0883: skl
7274 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7275 ILK_DPFC_DISABLE_DUMMY0);
7278 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7282 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7283 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7284 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7287 /* WaSwitchSolVfFArbitrationPriority:bdw */
7288 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7290 /* WaPsrDPAMaskVBlankInSRD:bdw */
7291 I915_WRITE(CHICKEN_PAR1_1,
7292 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7294 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7295 for_each_pipe(dev_priv, pipe) {
7296 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7297 I915_READ(CHICKEN_PIPESL_1(pipe)) |
7298 BDW_DPRS_MASK_VBLANK_SRD);
7301 /* WaVSRefCountFullforceMissDisable:bdw */
7302 /* WaDSRefCountFullforceMissDisable:bdw */
7303 I915_WRITE(GEN7_FF_THREAD_MODE,
7304 I915_READ(GEN7_FF_THREAD_MODE) &
7305 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7307 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7308 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7310 /* WaDisableSDEUnitClockGating:bdw */
7311 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7312 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7314 /* WaProgramL3SqcReg1Default:bdw */
7315 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7317 /* WaKVMNotificationOnConfigChange:bdw */
7318 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7319 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7321 lpt_init_clock_gating(dev_priv);
7323 /* WaDisableDopClockGating:bdw
7325 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7328 I915_WRITE(GEN6_UCGCTL1,
7329 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7332 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7334 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7335 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7336 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7339 /* This is required by WaCatErrorRejectionIssue:hsw */
7340 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7341 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7342 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7344 /* WaSwitchSolVfFArbitrationPriority:hsw */
7345 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7347 lpt_init_clock_gating(dev_priv);
7350 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7354 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7356 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7357 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7358 I915_READ(ILK_DISPLAY_CHICKEN1) |
7361 /* WaDisableBackToBackFlipFix:ivb */
7362 I915_WRITE(IVB_CHICKEN3,
7363 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7364 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7366 if (IS_IVB_GT1(dev_priv))
7367 I915_WRITE(GEN7_ROW_CHICKEN2,
7368 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7370 /* must write both registers */
7371 I915_WRITE(GEN7_ROW_CHICKEN2,
7372 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7373 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7374 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7378 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7379 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7381 I915_WRITE(GEN6_UCGCTL2,
7382 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7384 /* This is required by WaCatErrorRejectionIssue:ivb */
7385 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7386 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7387 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7389 g4x_disable_trickle_feed(dev_priv);
7391 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7392 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7393 snpcr |= GEN6_MBC_SNPCR_MED;
7394 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7396 if (!HAS_PCH_NOP(dev_priv))
7397 cpt_init_clock_gating(dev_priv);
7399 gen6_check_mch_setup(dev_priv);
7402 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7404 /* WaDisableBackToBackFlipFix:vlv */
7405 I915_WRITE(IVB_CHICKEN3,
7406 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7407 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7409 /* WaDisableDopClockGating:vlv */
7410 I915_WRITE(GEN7_ROW_CHICKEN2,
7411 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7413 /* This is required by WaCatErrorRejectionIssue:vlv */
7414 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7415 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7416 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7419 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7420 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7422 I915_WRITE(GEN6_UCGCTL2,
7423 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7425 /* WaDisableL3Bank2xClockGate:vlv
7426 * Disabling L3 clock gating- MMIO 940c[25] = 1
7427 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7428 I915_WRITE(GEN7_UCGCTL4,
7429 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7432 * WaDisableVLVClockGating_VBIIssue:vlv
7433 * Disable clock gating on th GCFG unit to prevent a delay
7434 * in the reporting of vblank events.
7436 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7439 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7441 /* WaVSRefCountFullforceMissDisable:chv */
7442 /* WaDSRefCountFullforceMissDisable:chv */
7443 I915_WRITE(GEN7_FF_THREAD_MODE,
7444 I915_READ(GEN7_FF_THREAD_MODE) &
7445 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7447 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7448 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7449 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7451 /* WaDisableCSUnitClockGating:chv */
7452 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7453 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7455 /* WaDisableSDEUnitClockGating:chv */
7456 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7457 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7460 * WaProgramL3SqcReg1Default:chv
7461 * See gfxspecs/Related Documents/Performance Guide/
7462 * LSQC Setting Recommendations.
7464 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7467 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7471 I915_WRITE(RENCLK_GATE_D1, 0);
7472 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7473 GS_UNIT_CLOCK_GATE_DISABLE |
7474 CL_UNIT_CLOCK_GATE_DISABLE);
7475 I915_WRITE(RAMCLK_GATE_D, 0);
7476 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7477 OVRUNIT_CLOCK_GATE_DISABLE |
7478 OVCUNIT_CLOCK_GATE_DISABLE;
7479 if (IS_GM45(dev_priv))
7480 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7481 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7483 g4x_disable_trickle_feed(dev_priv);
7486 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7488 struct intel_uncore *uncore = &dev_priv->uncore;
7490 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7491 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7492 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7493 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7494 intel_uncore_write16(uncore, DEUC, 0);
7495 intel_uncore_write(uncore,
7497 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7500 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7502 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7503 I965_RCC_CLOCK_GATE_DISABLE |
7504 I965_RCPB_CLOCK_GATE_DISABLE |
7505 I965_ISC_CLOCK_GATE_DISABLE |
7506 I965_FBC_CLOCK_GATE_DISABLE);
7507 I915_WRITE(RENCLK_GATE_D2, 0);
7508 I915_WRITE(MI_ARB_STATE,
7509 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7512 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7514 u32 dstate = I915_READ(D_STATE);
7516 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7517 DSTATE_DOT_CLOCK_GATING;
7518 I915_WRITE(D_STATE, dstate);
7520 if (IS_PINEVIEW(dev_priv))
7521 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7523 /* IIR "flip pending" means done if this bit is set */
7524 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7526 /* interrupts should cause a wake up from C3 */
7527 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7529 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7530 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7532 I915_WRITE(MI_ARB_STATE,
7533 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7536 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7538 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7540 /* interrupts should cause a wake up from C3 */
7541 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7542 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7544 I915_WRITE(MEM_MODE,
7545 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7548 * Have FBC ignore 3D activity since we use software
7549 * render tracking, and otherwise a pure 3D workload
7550 * (even if it just renders a single frame and then does
7551 * abosultely nothing) would not allow FBC to recompress
7552 * until a 2D blit occurs.
7555 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7558 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7560 I915_WRITE(MEM_MODE,
7561 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7562 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7565 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7567 dev_priv->display.init_clock_gating(dev_priv);
7570 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7572 if (HAS_PCH_LPT(dev_priv))
7573 lpt_suspend_hw(dev_priv);
7576 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7578 drm_dbg_kms(&dev_priv->drm,
7579 "No clock gating settings or workarounds applied.\n");
7583 * intel_init_clock_gating_hooks - setup the clock gating hooks
7584 * @dev_priv: device private
7586 * Setup the hooks that configure which clocks of a given platform can be
7587 * gated and also apply various GT and display specific workarounds for these
7588 * platforms. Note that some GT specific workarounds are applied separately
7589 * when GPU contexts or batchbuffers start their execution.
7591 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7593 if (IS_GEN(dev_priv, 12))
7594 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7595 else if (IS_GEN(dev_priv, 11))
7596 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7597 else if (IS_CANNONLAKE(dev_priv))
7598 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7599 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7600 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7601 else if (IS_SKYLAKE(dev_priv))
7602 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7603 else if (IS_KABYLAKE(dev_priv))
7604 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7605 else if (IS_BROXTON(dev_priv))
7606 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7607 else if (IS_GEMINILAKE(dev_priv))
7608 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7609 else if (IS_BROADWELL(dev_priv))
7610 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7611 else if (IS_CHERRYVIEW(dev_priv))
7612 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7613 else if (IS_HASWELL(dev_priv))
7614 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7615 else if (IS_IVYBRIDGE(dev_priv))
7616 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7617 else if (IS_VALLEYVIEW(dev_priv))
7618 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7619 else if (IS_GEN(dev_priv, 6))
7620 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7621 else if (IS_GEN(dev_priv, 5))
7622 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7623 else if (IS_G4X(dev_priv))
7624 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7625 else if (IS_I965GM(dev_priv))
7626 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7627 else if (IS_I965G(dev_priv))
7628 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7629 else if (IS_GEN(dev_priv, 3))
7630 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7631 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7632 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7633 else if (IS_GEN(dev_priv, 2))
7634 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7636 MISSING_CASE(INTEL_DEVID(dev_priv));
7637 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7641 /* Set up chip specific power management-related functions */
7642 void intel_init_pm(struct drm_i915_private *dev_priv)
7645 if (IS_PINEVIEW(dev_priv))
7646 pnv_get_mem_freq(dev_priv);
7647 else if (IS_GEN(dev_priv, 5))
7648 ilk_get_mem_freq(dev_priv);
7650 if (intel_has_sagv(dev_priv))
7651 skl_setup_sagv_block_time(dev_priv);
7653 /* For FIFO watermark updates */
7654 if (INTEL_GEN(dev_priv) >= 9) {
7655 skl_setup_wm_latency(dev_priv);
7656 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7657 } else if (HAS_PCH_SPLIT(dev_priv)) {
7658 ilk_setup_wm_latency(dev_priv);
7660 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7661 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7662 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7663 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7664 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7665 dev_priv->display.compute_intermediate_wm =
7666 ilk_compute_intermediate_wm;
7667 dev_priv->display.initial_watermarks =
7668 ilk_initial_watermarks;
7669 dev_priv->display.optimize_watermarks =
7670 ilk_optimize_watermarks;
7672 drm_dbg_kms(&dev_priv->drm,
7673 "Failed to read display plane latency. "
7676 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7677 vlv_setup_wm_latency(dev_priv);
7678 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7679 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7680 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7681 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7682 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7683 } else if (IS_G4X(dev_priv)) {
7684 g4x_setup_wm_latency(dev_priv);
7685 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7686 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7687 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7688 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7689 } else if (IS_PINEVIEW(dev_priv)) {
7690 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7693 dev_priv->mem_freq)) {
7694 drm_info(&dev_priv->drm,
7695 "failed to find known CxSR latency "
7696 "(found ddr%s fsb freq %d, mem freq %d), "
7698 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7699 dev_priv->fsb_freq, dev_priv->mem_freq);
7700 /* Disable CxSR and never update its watermark again */
7701 intel_set_memory_cxsr(dev_priv, false);
7702 dev_priv->display.update_wm = NULL;
7704 dev_priv->display.update_wm = pnv_update_wm;
7705 } else if (IS_GEN(dev_priv, 4)) {
7706 dev_priv->display.update_wm = i965_update_wm;
7707 } else if (IS_GEN(dev_priv, 3)) {
7708 dev_priv->display.update_wm = i9xx_update_wm;
7709 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7710 } else if (IS_GEN(dev_priv, 2)) {
7711 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7712 dev_priv->display.update_wm = i845_update_wm;
7713 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7715 dev_priv->display.update_wm = i9xx_update_wm;
7716 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7719 drm_err(&dev_priv->drm,
7720 "unexpected fall-through in %s\n", __func__);
7724 void intel_pm_setup(struct drm_i915_private *dev_priv)
7726 dev_priv->runtime_pm.suspended = false;
7727 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7730 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7732 struct intel_dbuf_state *dbuf_state;
7734 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7738 return &dbuf_state->base;
7741 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7742 struct intel_global_state *state)
7747 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7748 .atomic_duplicate_state = intel_dbuf_duplicate_state,
7749 .atomic_destroy_state = intel_dbuf_destroy_state,
7752 struct intel_dbuf_state *
7753 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7755 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7756 struct intel_global_state *dbuf_state;
7758 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7759 if (IS_ERR(dbuf_state))
7760 return ERR_CAST(dbuf_state);
7762 return to_intel_dbuf_state(dbuf_state);
7765 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7767 struct intel_dbuf_state *dbuf_state;
7769 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7773 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7774 &dbuf_state->base, &intel_dbuf_funcs);
7779 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7781 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7782 const struct intel_dbuf_state *new_dbuf_state =
7783 intel_atomic_get_new_dbuf_state(state);
7784 const struct intel_dbuf_state *old_dbuf_state =
7785 intel_atomic_get_old_dbuf_state(state);
7787 if (!new_dbuf_state ||
7788 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7791 WARN_ON(!new_dbuf_state->base.changed);
7793 gen9_dbuf_slices_update(dev_priv,
7794 old_dbuf_state->enabled_slices |
7795 new_dbuf_state->enabled_slices);
7798 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7800 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7801 const struct intel_dbuf_state *new_dbuf_state =
7802 intel_atomic_get_new_dbuf_state(state);
7803 const struct intel_dbuf_state *old_dbuf_state =
7804 intel_atomic_get_old_dbuf_state(state);
7806 if (!new_dbuf_state ||
7807 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7810 WARN_ON(!new_dbuf_state->base.changed);
7812 gen9_dbuf_slices_update(dev_priv,
7813 new_dbuf_state->enabled_slices);