2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
35 #include "display/intel_atomic.h"
36 #include "display/intel_bw.h"
37 #include "display/intel_display_types.h"
38 #include "display/intel_fbc.h"
39 #include "display/intel_sprite.h"
41 #include "gt/intel_llc.h"
44 #include "i915_fixed.h"
46 #include "i915_trace.h"
48 #include "intel_sideband.h"
49 #include "../../../platform/x86/intel_ips.h"
51 /* Stores plane specific WM parameters */
52 struct skl_wm_params {
53 bool x_tiled, y_tiled;
60 u32 plane_bytes_per_line;
61 uint_fixed_16_16_t plane_blocks_per_line;
62 uint_fixed_16_16_t y_tile_minimum;
67 /* used in computing the new watermarks state */
68 struct intel_wm_config {
69 unsigned int num_pipes_active;
74 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
76 if (HAS_LLC(dev_priv)) {
78 * WaCompressedResourceDisplayNewHashMode:skl,kbl
79 * Display WA #0390: skl,kbl
81 * Must match Sampler, Pixel Back End, and Media. See
82 * WaCompressedResourceSamplerPbeMediaNewHashMode.
84 I915_WRITE(CHICKEN_PAR1_1,
85 I915_READ(CHICKEN_PAR1_1) |
86 SKL_DE_COMPRESSED_HASH_MODE);
89 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
90 I915_WRITE(CHICKEN_PAR1_1,
91 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
93 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
94 I915_WRITE(GEN8_CHICKEN_DCPR_1,
95 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
97 /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
98 /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
99 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
101 DISP_FBC_MEMORY_WAKE);
103 /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
104 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
105 ILK_DPFC_DISABLE_DUMMY0);
107 if (IS_SKYLAKE(dev_priv)) {
108 /* WaDisableDopClockGating */
109 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
110 & ~GEN7_DOP_CLOCK_GATE_ENABLE);
114 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
116 gen9_init_clock_gating(dev_priv);
118 /* WaDisableSDEUnitClockGating:bxt */
119 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
120 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
124 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
126 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
127 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
130 * Wa: Backlight PWM may stop in the asserted state, causing backlight
133 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
134 PWM1_GATING_DIS | PWM2_GATING_DIS);
137 * Lower the display internal timeout.
138 * This is needed to avoid any hard hangs when DSI port PLL
139 * is off and a MMIO access is attempted by any privilege
140 * application, using batch buffers or any other means.
142 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
145 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
147 gen9_init_clock_gating(dev_priv);
150 * WaDisablePWMClockGating:glk
151 * Backlight PWM may stop in the asserted state, causing backlight
154 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
155 PWM1_GATING_DIS | PWM2_GATING_DIS);
158 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
162 tmp = I915_READ(CLKCFG);
164 switch (tmp & CLKCFG_FSB_MASK) {
166 dev_priv->fsb_freq = 533; /* 133*4 */
169 dev_priv->fsb_freq = 800; /* 200*4 */
172 dev_priv->fsb_freq = 667; /* 167*4 */
175 dev_priv->fsb_freq = 400; /* 100*4 */
179 switch (tmp & CLKCFG_MEM_MASK) {
181 dev_priv->mem_freq = 533;
184 dev_priv->mem_freq = 667;
187 dev_priv->mem_freq = 800;
191 /* detect pineview DDR3 setting */
192 tmp = I915_READ(CSHRDDR3CTL);
193 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
196 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
200 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
201 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
203 switch (ddrpll & 0xff) {
205 dev_priv->mem_freq = 800;
208 dev_priv->mem_freq = 1066;
211 dev_priv->mem_freq = 1333;
214 dev_priv->mem_freq = 1600;
217 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
219 dev_priv->mem_freq = 0;
223 switch (csipll & 0x3ff) {
225 dev_priv->fsb_freq = 3200;
228 dev_priv->fsb_freq = 3733;
231 dev_priv->fsb_freq = 4266;
234 dev_priv->fsb_freq = 4800;
237 dev_priv->fsb_freq = 5333;
240 dev_priv->fsb_freq = 5866;
243 dev_priv->fsb_freq = 6400;
246 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
248 dev_priv->fsb_freq = 0;
253 static const struct cxsr_latency cxsr_latency_table[] = {
254 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
255 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
256 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
257 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
258 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
260 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
261 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
262 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
263 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
264 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
266 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
267 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
268 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
269 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
270 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
272 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
273 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
274 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
275 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
276 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
278 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
279 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
280 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
281 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
282 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
284 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
285 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
286 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
287 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
288 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
291 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
296 const struct cxsr_latency *latency;
299 if (fsb == 0 || mem == 0)
302 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
303 latency = &cxsr_latency_table[i];
304 if (is_desktop == latency->is_desktop &&
305 is_ddr3 == latency->is_ddr3 &&
306 fsb == latency->fsb_freq && mem == latency->mem_freq)
310 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
315 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
319 vlv_punit_get(dev_priv);
321 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
323 val &= ~FORCE_DDR_HIGH_FREQ;
325 val |= FORCE_DDR_HIGH_FREQ;
326 val &= ~FORCE_DDR_LOW_FREQ;
327 val |= FORCE_DDR_FREQ_REQ_ACK;
328 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
330 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
331 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
332 drm_err(&dev_priv->drm,
333 "timed out waiting for Punit DDR DVFS request\n");
335 vlv_punit_put(dev_priv);
338 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
342 vlv_punit_get(dev_priv);
344 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
346 val |= DSP_MAXFIFO_PM5_ENABLE;
348 val &= ~DSP_MAXFIFO_PM5_ENABLE;
349 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
351 vlv_punit_put(dev_priv);
354 #define FW_WM(value, plane) \
355 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
357 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
362 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
363 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
364 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
365 POSTING_READ(FW_BLC_SELF_VLV);
366 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
367 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
368 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
369 POSTING_READ(FW_BLC_SELF);
370 } else if (IS_PINEVIEW(dev_priv)) {
371 val = I915_READ(DSPFW3);
372 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
374 val |= PINEVIEW_SELF_REFRESH_EN;
376 val &= ~PINEVIEW_SELF_REFRESH_EN;
377 I915_WRITE(DSPFW3, val);
378 POSTING_READ(DSPFW3);
379 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
380 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
381 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
382 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
383 I915_WRITE(FW_BLC_SELF, val);
384 POSTING_READ(FW_BLC_SELF);
385 } else if (IS_I915GM(dev_priv)) {
387 * FIXME can't find a bit like this for 915G, and
388 * and yet it does have the related watermark in
389 * FW_BLC_SELF. What's going on?
391 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
392 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
393 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
394 I915_WRITE(INSTPM, val);
395 POSTING_READ(INSTPM);
400 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
402 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
403 enableddisabled(enable),
404 enableddisabled(was_enabled));
410 * intel_set_memory_cxsr - Configure CxSR state
411 * @dev_priv: i915 device
412 * @enable: Allow vs. disallow CxSR
414 * Allow or disallow the system to enter a special CxSR
415 * (C-state self refresh) state. What typically happens in CxSR mode
416 * is that several display FIFOs may get combined into a single larger
417 * FIFO for a particular plane (so called max FIFO mode) to allow the
418 * system to defer memory fetches longer, and the memory will enter
421 * Note that enabling CxSR does not guarantee that the system enter
422 * this special mode, nor does it guarantee that the system stays
423 * in that mode once entered. So this just allows/disallows the system
424 * to autonomously utilize the CxSR mode. Other factors such as core
425 * C-states will affect when/if the system actually enters/exits the
428 * Note that on VLV/CHV this actually only controls the max FIFO mode,
429 * and the system is free to enter/exit memory self refresh at any time
430 * even when the use of CxSR has been disallowed.
432 * While the system is actually in the CxSR/max FIFO mode, some plane
433 * control registers will not get latched on vblank. Thus in order to
434 * guarantee the system will respond to changes in the plane registers
435 * we must always disallow CxSR prior to making changes to those registers.
436 * Unfortunately the system will re-evaluate the CxSR conditions at
437 * frame start which happens after vblank start (which is when the plane
438 * registers would get latched), so we can't proceed with the plane update
439 * during the same frame where we disallowed CxSR.
441 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
442 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
443 * the hardware w.r.t. HPLL SR when writing to plane registers.
444 * Disallowing just CxSR is sufficient.
446 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
450 mutex_lock(&dev_priv->wm.wm_mutex);
451 ret = _intel_set_memory_cxsr(dev_priv, enable);
452 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
453 dev_priv->wm.vlv.cxsr = enable;
454 else if (IS_G4X(dev_priv))
455 dev_priv->wm.g4x.cxsr = enable;
456 mutex_unlock(&dev_priv->wm.wm_mutex);
462 * Latency for FIFO fetches is dependent on several factors:
463 * - memory configuration (speed, channels)
465 * - current MCH state
466 * It can be fairly high in some situations, so here we assume a fairly
467 * pessimal value. It's a tradeoff between extra memory fetches (if we
468 * set this value too high, the FIFO will fetch frequently to stay full)
469 * and power consumption (set it too low to save power and we might see
470 * FIFO underruns and display "flicker").
472 * A value of 5us seems to be a good balance; safe for very low end
473 * platforms but not overly aggressive on lower latency configs.
475 static const int pessimal_latency_ns = 5000;
477 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
478 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
480 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
482 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
483 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
484 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
485 enum pipe pipe = crtc->pipe;
486 int sprite0_start, sprite1_start;
487 u32 dsparb, dsparb2, dsparb3;
491 dsparb = I915_READ(DSPARB);
492 dsparb2 = I915_READ(DSPARB2);
493 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
494 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
497 dsparb = I915_READ(DSPARB);
498 dsparb2 = I915_READ(DSPARB2);
499 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
500 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
503 dsparb2 = I915_READ(DSPARB2);
504 dsparb3 = I915_READ(DSPARB3);
505 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
506 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
513 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
514 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
515 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
516 fifo_state->plane[PLANE_CURSOR] = 63;
519 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
520 enum i9xx_plane_id i9xx_plane)
522 u32 dsparb = I915_READ(DSPARB);
525 size = dsparb & 0x7f;
526 if (i9xx_plane == PLANE_B)
527 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
529 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
530 dsparb, plane_name(i9xx_plane), size);
535 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
536 enum i9xx_plane_id i9xx_plane)
538 u32 dsparb = I915_READ(DSPARB);
541 size = dsparb & 0x1ff;
542 if (i9xx_plane == PLANE_B)
543 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
544 size >>= 1; /* Convert to cachelines */
546 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
547 dsparb, plane_name(i9xx_plane), size);
552 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
553 enum i9xx_plane_id i9xx_plane)
555 u32 dsparb = I915_READ(DSPARB);
558 size = dsparb & 0x7f;
559 size >>= 2; /* Convert to cachelines */
561 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
562 dsparb, plane_name(i9xx_plane), size);
567 /* Pineview has different values for various configs */
568 static const struct intel_watermark_params pnv_display_wm = {
569 .fifo_size = PINEVIEW_DISPLAY_FIFO,
570 .max_wm = PINEVIEW_MAX_WM,
571 .default_wm = PINEVIEW_DFT_WM,
572 .guard_size = PINEVIEW_GUARD_WM,
573 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
576 static const struct intel_watermark_params pnv_display_hplloff_wm = {
577 .fifo_size = PINEVIEW_DISPLAY_FIFO,
578 .max_wm = PINEVIEW_MAX_WM,
579 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
580 .guard_size = PINEVIEW_GUARD_WM,
581 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
584 static const struct intel_watermark_params pnv_cursor_wm = {
585 .fifo_size = PINEVIEW_CURSOR_FIFO,
586 .max_wm = PINEVIEW_CURSOR_MAX_WM,
587 .default_wm = PINEVIEW_CURSOR_DFT_WM,
588 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
589 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
592 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
593 .fifo_size = PINEVIEW_CURSOR_FIFO,
594 .max_wm = PINEVIEW_CURSOR_MAX_WM,
595 .default_wm = PINEVIEW_CURSOR_DFT_WM,
596 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
597 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
600 static const struct intel_watermark_params i965_cursor_wm_info = {
601 .fifo_size = I965_CURSOR_FIFO,
602 .max_wm = I965_CURSOR_MAX_WM,
603 .default_wm = I965_CURSOR_DFT_WM,
605 .cacheline_size = I915_FIFO_LINE_SIZE,
608 static const struct intel_watermark_params i945_wm_info = {
609 .fifo_size = I945_FIFO_SIZE,
610 .max_wm = I915_MAX_WM,
613 .cacheline_size = I915_FIFO_LINE_SIZE,
616 static const struct intel_watermark_params i915_wm_info = {
617 .fifo_size = I915_FIFO_SIZE,
618 .max_wm = I915_MAX_WM,
621 .cacheline_size = I915_FIFO_LINE_SIZE,
624 static const struct intel_watermark_params i830_a_wm_info = {
625 .fifo_size = I855GM_FIFO_SIZE,
626 .max_wm = I915_MAX_WM,
629 .cacheline_size = I830_FIFO_LINE_SIZE,
632 static const struct intel_watermark_params i830_bc_wm_info = {
633 .fifo_size = I855GM_FIFO_SIZE,
634 .max_wm = I915_MAX_WM/2,
637 .cacheline_size = I830_FIFO_LINE_SIZE,
640 static const struct intel_watermark_params i845_wm_info = {
641 .fifo_size = I830_FIFO_SIZE,
642 .max_wm = I915_MAX_WM,
645 .cacheline_size = I830_FIFO_LINE_SIZE,
649 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
650 * @pixel_rate: Pipe pixel rate in kHz
651 * @cpp: Plane bytes per pixel
652 * @latency: Memory wakeup latency in 0.1us units
654 * Compute the watermark using the method 1 or "small buffer"
655 * formula. The caller may additonally add extra cachelines
656 * to account for TLB misses and clock crossings.
658 * This method is concerned with the short term drain rate
659 * of the FIFO, ie. it does not account for blanking periods
660 * which would effectively reduce the average drain rate across
661 * a longer period. The name "small" refers to the fact the
662 * FIFO is relatively small compared to the amount of data
665 * The FIFO level vs. time graph might look something like:
669 * __---__---__ (- plane active, _ blanking)
672 * or perhaps like this:
675 * __----__----__ (- plane active, _ blanking)
679 * The watermark in bytes
681 static unsigned int intel_wm_method1(unsigned int pixel_rate,
683 unsigned int latency)
687 ret = mul_u32_u32(pixel_rate, cpp * latency);
688 ret = DIV_ROUND_UP_ULL(ret, 10000);
694 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
695 * @pixel_rate: Pipe pixel rate in kHz
696 * @htotal: Pipe horizontal total
697 * @width: Plane width in pixels
698 * @cpp: Plane bytes per pixel
699 * @latency: Memory wakeup latency in 0.1us units
701 * Compute the watermark using the method 2 or "large buffer"
702 * formula. The caller may additonally add extra cachelines
703 * to account for TLB misses and clock crossings.
705 * This method is concerned with the long term drain rate
706 * of the FIFO, ie. it does account for blanking periods
707 * which effectively reduce the average drain rate across
708 * a longer period. The name "large" refers to the fact the
709 * FIFO is relatively large compared to the amount of data
712 * The FIFO level vs. time graph might look something like:
717 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
721 * The watermark in bytes
723 static unsigned int intel_wm_method2(unsigned int pixel_rate,
727 unsigned int latency)
732 * FIXME remove once all users are computing
733 * watermarks in the correct place.
735 if (WARN_ON_ONCE(htotal == 0))
738 ret = (latency * pixel_rate) / (htotal * 10000);
739 ret = (ret + 1) * width * cpp;
745 * intel_calculate_wm - calculate watermark level
746 * @pixel_rate: pixel clock
747 * @wm: chip FIFO params
748 * @fifo_size: size of the FIFO buffer
749 * @cpp: bytes per pixel
750 * @latency_ns: memory latency for the platform
752 * Calculate the watermark level (the level at which the display plane will
753 * start fetching from memory again). Each chip has a different display
754 * FIFO size and allocation, so the caller needs to figure that out and pass
755 * in the correct intel_watermark_params structure.
757 * As the pixel clock runs, the FIFO will be drained at a rate that depends
758 * on the pixel size. When it reaches the watermark level, it'll start
759 * fetching FIFO line sized based chunks from memory until the FIFO fills
760 * past the watermark point. If the FIFO drains completely, a FIFO underrun
761 * will occur, and a display engine hang could result.
763 static unsigned int intel_calculate_wm(int pixel_rate,
764 const struct intel_watermark_params *wm,
765 int fifo_size, int cpp,
766 unsigned int latency_ns)
768 int entries, wm_size;
771 * Note: we need to make sure we don't overflow for various clock &
773 * clocks go from a few thousand to several hundred thousand.
774 * latency is usually a few thousand
776 entries = intel_wm_method1(pixel_rate, cpp,
778 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
780 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
782 wm_size = fifo_size - entries;
783 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
785 /* Don't promote wm_size to unsigned... */
786 if (wm_size > wm->max_wm)
787 wm_size = wm->max_wm;
789 wm_size = wm->default_wm;
792 * Bspec seems to indicate that the value shouldn't be lower than
793 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
794 * Lets go for 8 which is the burst size since certain platforms
795 * already use a hardcoded 8 (which is what the spec says should be
804 static bool is_disabling(int old, int new, int threshold)
806 return old >= threshold && new < threshold;
809 static bool is_enabling(int old, int new, int threshold)
811 return old < threshold && new >= threshold;
814 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
816 return dev_priv->wm.max_level + 1;
819 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
820 const struct intel_plane_state *plane_state)
822 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
824 /* FIXME check the 'enable' instead */
825 if (!crtc_state->hw.active)
829 * Treat cursor with fb as always visible since cursor updates
830 * can happen faster than the vrefresh rate, and the current
831 * watermark code doesn't handle that correctly. Cursor updates
832 * which set/clear the fb or change the cursor size are going
833 * to get throttled by intel_legacy_cursor_update() to work
834 * around this problem with the watermark code.
836 if (plane->id == PLANE_CURSOR)
837 return plane_state->hw.fb != NULL;
839 return plane_state->uapi.visible;
842 static bool intel_crtc_active(struct intel_crtc *crtc)
844 /* Be paranoid as we can arrive here with only partial
845 * state retrieved from the hardware during setup.
847 * We can ditch the adjusted_mode.crtc_clock check as soon
848 * as Haswell has gained clock readout/fastboot support.
850 * We can ditch the crtc->primary->state->fb check as soon as we can
851 * properly reconstruct framebuffers.
853 * FIXME: The intel_crtc->active here should be switched to
854 * crtc->state->active once we have proper CRTC states wired up
857 return crtc->active && crtc->base.primary->state->fb &&
858 crtc->config->hw.adjusted_mode.crtc_clock;
861 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
863 struct intel_crtc *crtc, *enabled = NULL;
865 for_each_intel_crtc(&dev_priv->drm, crtc) {
866 if (intel_crtc_active(crtc)) {
876 static void pnv_update_wm(struct intel_crtc *unused_crtc)
878 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
879 struct intel_crtc *crtc;
880 const struct cxsr_latency *latency;
884 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
889 drm_dbg_kms(&dev_priv->drm,
890 "Unknown FSB/MEM found, disable CxSR\n");
891 intel_set_memory_cxsr(dev_priv, false);
895 crtc = single_enabled_crtc(dev_priv);
897 const struct drm_display_mode *adjusted_mode =
898 &crtc->config->hw.adjusted_mode;
899 const struct drm_framebuffer *fb =
900 crtc->base.primary->state->fb;
901 int cpp = fb->format->cpp[0];
902 int clock = adjusted_mode->crtc_clock;
905 wm = intel_calculate_wm(clock, &pnv_display_wm,
906 pnv_display_wm.fifo_size,
907 cpp, latency->display_sr);
908 reg = I915_READ(DSPFW1);
909 reg &= ~DSPFW_SR_MASK;
910 reg |= FW_WM(wm, SR);
911 I915_WRITE(DSPFW1, reg);
912 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
915 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
916 pnv_display_wm.fifo_size,
917 4, latency->cursor_sr);
918 reg = I915_READ(DSPFW3);
919 reg &= ~DSPFW_CURSOR_SR_MASK;
920 reg |= FW_WM(wm, CURSOR_SR);
921 I915_WRITE(DSPFW3, reg);
923 /* Display HPLL off SR */
924 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
925 pnv_display_hplloff_wm.fifo_size,
926 cpp, latency->display_hpll_disable);
927 reg = I915_READ(DSPFW3);
928 reg &= ~DSPFW_HPLL_SR_MASK;
929 reg |= FW_WM(wm, HPLL_SR);
930 I915_WRITE(DSPFW3, reg);
932 /* cursor HPLL off SR */
933 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
934 pnv_display_hplloff_wm.fifo_size,
935 4, latency->cursor_hpll_disable);
936 reg = I915_READ(DSPFW3);
937 reg &= ~DSPFW_HPLL_CURSOR_MASK;
938 reg |= FW_WM(wm, HPLL_CURSOR);
939 I915_WRITE(DSPFW3, reg);
940 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
942 intel_set_memory_cxsr(dev_priv, true);
944 intel_set_memory_cxsr(dev_priv, false);
949 * Documentation says:
950 * "If the line size is small, the TLB fetches can get in the way of the
951 * data fetches, causing some lag in the pixel data return which is not
952 * accounted for in the above formulas. The following adjustment only
953 * needs to be applied if eight whole lines fit in the buffer at once.
954 * The WM is adjusted upwards by the difference between the FIFO size
955 * and the size of 8 whole lines. This adjustment is always performed
956 * in the actual pixel depth regardless of whether FBC is enabled or not."
958 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
960 int tlb_miss = fifo_size * 64 - width * cpp * 8;
962 return max(0, tlb_miss);
965 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
966 const struct g4x_wm_values *wm)
970 for_each_pipe(dev_priv, pipe)
971 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
974 FW_WM(wm->sr.plane, SR) |
975 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
976 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
977 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
979 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
980 FW_WM(wm->sr.fbc, FBC_SR) |
981 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
982 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
983 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
984 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
986 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
987 FW_WM(wm->sr.cursor, CURSOR_SR) |
988 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
989 FW_WM(wm->hpll.plane, HPLL_SR));
991 POSTING_READ(DSPFW1);
994 #define FW_WM_VLV(value, plane) \
995 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
997 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
998 const struct vlv_wm_values *wm)
1002 for_each_pipe(dev_priv, pipe) {
1003 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1005 I915_WRITE(VLV_DDL(pipe),
1006 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1007 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1008 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1009 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1013 * Zero the (unused) WM1 watermarks, and also clear all the
1014 * high order bits so that there are no out of bounds values
1015 * present in the registers during the reprogramming.
1017 I915_WRITE(DSPHOWM, 0);
1018 I915_WRITE(DSPHOWM1, 0);
1019 I915_WRITE(DSPFW4, 0);
1020 I915_WRITE(DSPFW5, 0);
1021 I915_WRITE(DSPFW6, 0);
1024 FW_WM(wm->sr.plane, SR) |
1025 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1026 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1027 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1029 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1030 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1031 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1033 FW_WM(wm->sr.cursor, CURSOR_SR));
1035 if (IS_CHERRYVIEW(dev_priv)) {
1036 I915_WRITE(DSPFW7_CHV,
1037 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1038 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1039 I915_WRITE(DSPFW8_CHV,
1040 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1041 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1042 I915_WRITE(DSPFW9_CHV,
1043 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1044 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1046 FW_WM(wm->sr.plane >> 9, SR_HI) |
1047 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1048 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1049 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1050 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1051 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1052 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1053 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1054 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1055 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1058 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1059 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1061 FW_WM(wm->sr.plane >> 9, SR_HI) |
1062 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1063 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1064 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1065 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1066 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1067 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1070 POSTING_READ(DSPFW1);
1075 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1077 /* all latencies in usec */
1078 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1079 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1080 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1082 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1085 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1088 * DSPCNTR[13] supposedly controls whether the
1089 * primary plane can use the FIFO space otherwise
1090 * reserved for the sprite plane. It's not 100% clear
1091 * what the actual FIFO size is, but it looks like we
1092 * can happily set both primary and sprite watermarks
1093 * up to 127 cachelines. So that would seem to mean
1094 * that either DSPCNTR[13] doesn't do anything, or that
1095 * the total FIFO is >= 256 cachelines in size. Either
1096 * way, we don't seem to have to worry about this
1097 * repartitioning as the maximum watermark value the
1098 * register can hold for each plane is lower than the
1099 * minimum FIFO size.
1105 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1107 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1109 MISSING_CASE(plane_id);
1114 static int g4x_fbc_fifo_size(int level)
1117 case G4X_WM_LEVEL_SR:
1119 case G4X_WM_LEVEL_HPLL:
1122 MISSING_CASE(level);
1127 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1128 const struct intel_plane_state *plane_state,
1131 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1132 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1133 const struct drm_display_mode *adjusted_mode =
1134 &crtc_state->hw.adjusted_mode;
1135 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1136 unsigned int clock, htotal, cpp, width, wm;
1141 if (!intel_wm_plane_visible(crtc_state, plane_state))
1144 cpp = plane_state->hw.fb->format->cpp[0];
1147 * Not 100% sure which way ELK should go here as the
1148 * spec only says CL/CTG should assume 32bpp and BW
1149 * doesn't need to. But as these things followed the
1150 * mobile vs. desktop lines on gen3 as well, let's
1151 * assume ELK doesn't need this.
1153 * The spec also fails to list such a restriction for
1154 * the HPLL watermark, which seems a little strange.
1155 * Let's use 32bpp for the HPLL watermark as well.
1157 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1158 level != G4X_WM_LEVEL_NORMAL)
1161 clock = adjusted_mode->crtc_clock;
1162 htotal = adjusted_mode->crtc_htotal;
1164 width = drm_rect_width(&plane_state->uapi.dst);
1166 if (plane->id == PLANE_CURSOR) {
1167 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1168 } else if (plane->id == PLANE_PRIMARY &&
1169 level == G4X_WM_LEVEL_NORMAL) {
1170 wm = intel_wm_method1(clock, cpp, latency);
1172 unsigned int small, large;
1174 small = intel_wm_method1(clock, cpp, latency);
1175 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1177 wm = min(small, large);
1180 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1183 wm = DIV_ROUND_UP(wm, 64) + 2;
1185 return min_t(unsigned int, wm, USHRT_MAX);
1188 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1189 int level, enum plane_id plane_id, u16 value)
1191 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1194 for (; level < intel_wm_num_levels(dev_priv); level++) {
1195 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1197 dirty |= raw->plane[plane_id] != value;
1198 raw->plane[plane_id] = value;
1204 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1205 int level, u16 value)
1207 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1210 /* NORMAL level doesn't have an FBC watermark */
1211 level = max(level, G4X_WM_LEVEL_SR);
1213 for (; level < intel_wm_num_levels(dev_priv); level++) {
1214 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1216 dirty |= raw->fbc != value;
1223 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1224 const struct intel_plane_state *plane_state,
1227 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1228 const struct intel_plane_state *plane_state)
1230 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1231 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1232 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1233 enum plane_id plane_id = plane->id;
1237 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1238 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1239 if (plane_id == PLANE_PRIMARY)
1240 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1244 for (level = 0; level < num_levels; level++) {
1245 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1248 wm = g4x_compute_wm(crtc_state, plane_state, level);
1249 max_wm = g4x_plane_fifo_size(plane_id, level);
1254 dirty |= raw->plane[plane_id] != wm;
1255 raw->plane[plane_id] = wm;
1257 if (plane_id != PLANE_PRIMARY ||
1258 level == G4X_WM_LEVEL_NORMAL)
1261 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1262 raw->plane[plane_id]);
1263 max_wm = g4x_fbc_fifo_size(level);
1266 * FBC wm is not mandatory as we
1267 * can always just disable its use.
1272 dirty |= raw->fbc != wm;
1276 /* mark watermarks as invalid */
1277 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1279 if (plane_id == PLANE_PRIMARY)
1280 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1284 drm_dbg_kms(&dev_priv->drm,
1285 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1287 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1288 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1289 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1291 if (plane_id == PLANE_PRIMARY)
1292 drm_dbg_kms(&dev_priv->drm,
1293 "FBC watermarks: SR=%d, HPLL=%d\n",
1294 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1295 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1301 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1302 enum plane_id plane_id, int level)
1304 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1306 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1309 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1312 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1314 if (level > dev_priv->wm.max_level)
1317 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1318 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1319 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1322 /* mark all levels starting from 'level' as invalid */
1323 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1324 struct g4x_wm_state *wm_state, int level)
1326 if (level <= G4X_WM_LEVEL_NORMAL) {
1327 enum plane_id plane_id;
1329 for_each_plane_id_on_crtc(crtc, plane_id)
1330 wm_state->wm.plane[plane_id] = USHRT_MAX;
1333 if (level <= G4X_WM_LEVEL_SR) {
1334 wm_state->cxsr = false;
1335 wm_state->sr.cursor = USHRT_MAX;
1336 wm_state->sr.plane = USHRT_MAX;
1337 wm_state->sr.fbc = USHRT_MAX;
1340 if (level <= G4X_WM_LEVEL_HPLL) {
1341 wm_state->hpll_en = false;
1342 wm_state->hpll.cursor = USHRT_MAX;
1343 wm_state->hpll.plane = USHRT_MAX;
1344 wm_state->hpll.fbc = USHRT_MAX;
1348 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1351 if (level < G4X_WM_LEVEL_SR)
1354 if (level >= G4X_WM_LEVEL_SR &&
1355 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1358 if (level >= G4X_WM_LEVEL_HPLL &&
1359 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1365 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1367 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1368 struct intel_atomic_state *state =
1369 to_intel_atomic_state(crtc_state->uapi.state);
1370 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1371 int num_active_planes = hweight8(crtc_state->active_planes &
1372 ~BIT(PLANE_CURSOR));
1373 const struct g4x_pipe_wm *raw;
1374 const struct intel_plane_state *old_plane_state;
1375 const struct intel_plane_state *new_plane_state;
1376 struct intel_plane *plane;
1377 enum plane_id plane_id;
1379 unsigned int dirty = 0;
1381 for_each_oldnew_intel_plane_in_state(state, plane,
1383 new_plane_state, i) {
1384 if (new_plane_state->hw.crtc != &crtc->base &&
1385 old_plane_state->hw.crtc != &crtc->base)
1388 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1389 dirty |= BIT(plane->id);
1395 level = G4X_WM_LEVEL_NORMAL;
1396 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1399 raw = &crtc_state->wm.g4x.raw[level];
1400 for_each_plane_id_on_crtc(crtc, plane_id)
1401 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1403 level = G4X_WM_LEVEL_SR;
1404 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1407 raw = &crtc_state->wm.g4x.raw[level];
1408 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1409 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1410 wm_state->sr.fbc = raw->fbc;
1412 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1414 level = G4X_WM_LEVEL_HPLL;
1415 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1418 raw = &crtc_state->wm.g4x.raw[level];
1419 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1420 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1421 wm_state->hpll.fbc = raw->fbc;
1423 wm_state->hpll_en = wm_state->cxsr;
1428 if (level == G4X_WM_LEVEL_NORMAL)
1431 /* invalidate the higher levels */
1432 g4x_invalidate_wms(crtc, wm_state, level);
1435 * Determine if the FBC watermark(s) can be used. IF
1436 * this isn't the case we prefer to disable the FBC
1437 * watermark(s) rather than disable the SR/HPLL
1438 * level(s) entirely. 'level-1' is the highest valid
1441 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1446 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1448 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1449 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1450 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1451 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1452 struct intel_atomic_state *intel_state =
1453 to_intel_atomic_state(new_crtc_state->uapi.state);
1454 const struct intel_crtc_state *old_crtc_state =
1455 intel_atomic_get_old_crtc_state(intel_state, crtc);
1456 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1457 enum plane_id plane_id;
1459 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1460 *intermediate = *optimal;
1462 intermediate->cxsr = false;
1463 intermediate->hpll_en = false;
1467 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1468 !new_crtc_state->disable_cxsr;
1469 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1470 !new_crtc_state->disable_cxsr;
1471 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1473 for_each_plane_id_on_crtc(crtc, plane_id) {
1474 intermediate->wm.plane[plane_id] =
1475 max(optimal->wm.plane[plane_id],
1476 active->wm.plane[plane_id]);
1478 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1479 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1482 intermediate->sr.plane = max(optimal->sr.plane,
1484 intermediate->sr.cursor = max(optimal->sr.cursor,
1486 intermediate->sr.fbc = max(optimal->sr.fbc,
1489 intermediate->hpll.plane = max(optimal->hpll.plane,
1490 active->hpll.plane);
1491 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1492 active->hpll.cursor);
1493 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1496 drm_WARN_ON(&dev_priv->drm,
1497 (intermediate->sr.plane >
1498 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1499 intermediate->sr.cursor >
1500 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1501 intermediate->cxsr);
1502 drm_WARN_ON(&dev_priv->drm,
1503 (intermediate->sr.plane >
1504 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1505 intermediate->sr.cursor >
1506 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1507 intermediate->hpll_en);
1509 drm_WARN_ON(&dev_priv->drm,
1510 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1511 intermediate->fbc_en && intermediate->cxsr);
1512 drm_WARN_ON(&dev_priv->drm,
1513 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1514 intermediate->fbc_en && intermediate->hpll_en);
1518 * If our intermediate WM are identical to the final WM, then we can
1519 * omit the post-vblank programming; only update if it's different.
1521 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1522 new_crtc_state->wm.need_postvbl_update = true;
1527 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1528 struct g4x_wm_values *wm)
1530 struct intel_crtc *crtc;
1531 int num_active_pipes = 0;
1537 for_each_intel_crtc(&dev_priv->drm, crtc) {
1538 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1543 if (!wm_state->cxsr)
1545 if (!wm_state->hpll_en)
1546 wm->hpll_en = false;
1547 if (!wm_state->fbc_en)
1553 if (num_active_pipes != 1) {
1555 wm->hpll_en = false;
1559 for_each_intel_crtc(&dev_priv->drm, crtc) {
1560 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1561 enum pipe pipe = crtc->pipe;
1563 wm->pipe[pipe] = wm_state->wm;
1564 if (crtc->active && wm->cxsr)
1565 wm->sr = wm_state->sr;
1566 if (crtc->active && wm->hpll_en)
1567 wm->hpll = wm_state->hpll;
1571 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1573 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1574 struct g4x_wm_values new_wm = {};
1576 g4x_merge_wm(dev_priv, &new_wm);
1578 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1581 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1582 _intel_set_memory_cxsr(dev_priv, false);
1584 g4x_write_wm_values(dev_priv, &new_wm);
1586 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1587 _intel_set_memory_cxsr(dev_priv, true);
1592 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1593 struct intel_crtc *crtc)
1595 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1596 const struct intel_crtc_state *crtc_state =
1597 intel_atomic_get_new_crtc_state(state, crtc);
1599 mutex_lock(&dev_priv->wm.wm_mutex);
1600 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1601 g4x_program_watermarks(dev_priv);
1602 mutex_unlock(&dev_priv->wm.wm_mutex);
1605 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1606 struct intel_crtc *crtc)
1608 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1609 const struct intel_crtc_state *crtc_state =
1610 intel_atomic_get_new_crtc_state(state, crtc);
1612 if (!crtc_state->wm.need_postvbl_update)
1615 mutex_lock(&dev_priv->wm.wm_mutex);
1616 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1617 g4x_program_watermarks(dev_priv);
1618 mutex_unlock(&dev_priv->wm.wm_mutex);
1621 /* latency must be in 0.1us units. */
1622 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1623 unsigned int htotal,
1626 unsigned int latency)
1630 ret = intel_wm_method2(pixel_rate, htotal,
1631 width, cpp, latency);
1632 ret = DIV_ROUND_UP(ret, 64);
1637 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1639 /* all latencies in usec */
1640 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1642 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1644 if (IS_CHERRYVIEW(dev_priv)) {
1645 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1646 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1648 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1652 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1653 const struct intel_plane_state *plane_state,
1656 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1657 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1658 const struct drm_display_mode *adjusted_mode =
1659 &crtc_state->hw.adjusted_mode;
1660 unsigned int clock, htotal, cpp, width, wm;
1662 if (dev_priv->wm.pri_latency[level] == 0)
1665 if (!intel_wm_plane_visible(crtc_state, plane_state))
1668 cpp = plane_state->hw.fb->format->cpp[0];
1669 clock = adjusted_mode->crtc_clock;
1670 htotal = adjusted_mode->crtc_htotal;
1671 width = crtc_state->pipe_src_w;
1673 if (plane->id == PLANE_CURSOR) {
1675 * FIXME the formula gives values that are
1676 * too big for the cursor FIFO, and hence we
1677 * would never be able to use cursors. For
1678 * now just hardcode the watermark.
1682 wm = vlv_wm_method2(clock, htotal, width, cpp,
1683 dev_priv->wm.pri_latency[level] * 10);
1686 return min_t(unsigned int, wm, USHRT_MAX);
1689 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1691 return (active_planes & (BIT(PLANE_SPRITE0) |
1692 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1695 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1697 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1698 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1699 const struct g4x_pipe_wm *raw =
1700 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1701 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1702 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1703 int num_active_planes = hweight8(active_planes);
1704 const int fifo_size = 511;
1705 int fifo_extra, fifo_left = fifo_size;
1706 int sprite0_fifo_extra = 0;
1707 unsigned int total_rate;
1708 enum plane_id plane_id;
1711 * When enabling sprite0 after sprite1 has already been enabled
1712 * we tend to get an underrun unless sprite0 already has some
1713 * FIFO space allcoated. Hence we always allocate at least one
1714 * cacheline for sprite0 whenever sprite1 is enabled.
1716 * All other plane enable sequences appear immune to this problem.
1718 if (vlv_need_sprite0_fifo_workaround(active_planes))
1719 sprite0_fifo_extra = 1;
1721 total_rate = raw->plane[PLANE_PRIMARY] +
1722 raw->plane[PLANE_SPRITE0] +
1723 raw->plane[PLANE_SPRITE1] +
1726 if (total_rate > fifo_size)
1729 if (total_rate == 0)
1732 for_each_plane_id_on_crtc(crtc, plane_id) {
1735 if ((active_planes & BIT(plane_id)) == 0) {
1736 fifo_state->plane[plane_id] = 0;
1740 rate = raw->plane[plane_id];
1741 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1742 fifo_left -= fifo_state->plane[plane_id];
1745 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1746 fifo_left -= sprite0_fifo_extra;
1748 fifo_state->plane[PLANE_CURSOR] = 63;
1750 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1752 /* spread the remainder evenly */
1753 for_each_plane_id_on_crtc(crtc, plane_id) {
1759 if ((active_planes & BIT(plane_id)) == 0)
1762 plane_extra = min(fifo_extra, fifo_left);
1763 fifo_state->plane[plane_id] += plane_extra;
1764 fifo_left -= plane_extra;
1767 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1769 /* give it all to the first plane if none are active */
1770 if (active_planes == 0) {
1771 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1772 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1778 /* mark all levels starting from 'level' as invalid */
1779 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1780 struct vlv_wm_state *wm_state, int level)
1782 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1784 for (; level < intel_wm_num_levels(dev_priv); level++) {
1785 enum plane_id plane_id;
1787 for_each_plane_id_on_crtc(crtc, plane_id)
1788 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1790 wm_state->sr[level].cursor = USHRT_MAX;
1791 wm_state->sr[level].plane = USHRT_MAX;
1795 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1800 return fifo_size - wm;
1804 * Starting from 'level' set all higher
1805 * levels to 'value' in the "raw" watermarks.
1807 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1808 int level, enum plane_id plane_id, u16 value)
1810 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1811 int num_levels = intel_wm_num_levels(dev_priv);
1814 for (; level < num_levels; level++) {
1815 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1817 dirty |= raw->plane[plane_id] != value;
1818 raw->plane[plane_id] = value;
1824 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1825 const struct intel_plane_state *plane_state)
1827 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1828 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1829 enum plane_id plane_id = plane->id;
1830 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1834 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1835 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1839 for (level = 0; level < num_levels; level++) {
1840 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1841 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1842 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1847 dirty |= raw->plane[plane_id] != wm;
1848 raw->plane[plane_id] = wm;
1851 /* mark all higher levels as invalid */
1852 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1856 drm_dbg_kms(&dev_priv->drm,
1857 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1859 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1860 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1861 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1866 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1867 enum plane_id plane_id, int level)
1869 const struct g4x_pipe_wm *raw =
1870 &crtc_state->wm.vlv.raw[level];
1871 const struct vlv_fifo_state *fifo_state =
1872 &crtc_state->wm.vlv.fifo_state;
1874 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1877 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1879 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1880 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1881 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1882 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1885 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1887 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1888 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1889 struct intel_atomic_state *state =
1890 to_intel_atomic_state(crtc_state->uapi.state);
1891 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1892 const struct vlv_fifo_state *fifo_state =
1893 &crtc_state->wm.vlv.fifo_state;
1894 int num_active_planes = hweight8(crtc_state->active_planes &
1895 ~BIT(PLANE_CURSOR));
1896 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1897 const struct intel_plane_state *old_plane_state;
1898 const struct intel_plane_state *new_plane_state;
1899 struct intel_plane *plane;
1900 enum plane_id plane_id;
1902 unsigned int dirty = 0;
1904 for_each_oldnew_intel_plane_in_state(state, plane,
1906 new_plane_state, i) {
1907 if (new_plane_state->hw.crtc != &crtc->base &&
1908 old_plane_state->hw.crtc != &crtc->base)
1911 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1912 dirty |= BIT(plane->id);
1916 * DSPARB registers may have been reset due to the
1917 * power well being turned off. Make sure we restore
1918 * them to a consistent state even if no primary/sprite
1919 * planes are initially active.
1922 crtc_state->fifo_changed = true;
1927 /* cursor changes don't warrant a FIFO recompute */
1928 if (dirty & ~BIT(PLANE_CURSOR)) {
1929 const struct intel_crtc_state *old_crtc_state =
1930 intel_atomic_get_old_crtc_state(state, crtc);
1931 const struct vlv_fifo_state *old_fifo_state =
1932 &old_crtc_state->wm.vlv.fifo_state;
1934 ret = vlv_compute_fifo(crtc_state);
1938 if (needs_modeset ||
1939 memcmp(old_fifo_state, fifo_state,
1940 sizeof(*fifo_state)) != 0)
1941 crtc_state->fifo_changed = true;
1944 /* initially allow all levels */
1945 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1947 * Note that enabling cxsr with no primary/sprite planes
1948 * enabled can wedge the pipe. Hence we only allow cxsr
1949 * with exactly one enabled primary/sprite plane.
1951 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1953 for (level = 0; level < wm_state->num_levels; level++) {
1954 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1955 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1957 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1960 for_each_plane_id_on_crtc(crtc, plane_id) {
1961 wm_state->wm[level].plane[plane_id] =
1962 vlv_invert_wm_value(raw->plane[plane_id],
1963 fifo_state->plane[plane_id]);
1966 wm_state->sr[level].plane =
1967 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1968 raw->plane[PLANE_SPRITE0],
1969 raw->plane[PLANE_SPRITE1]),
1972 wm_state->sr[level].cursor =
1973 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1980 /* limit to only levels we can actually handle */
1981 wm_state->num_levels = level;
1983 /* invalidate the higher levels */
1984 vlv_invalidate_wms(crtc, wm_state, level);
1989 #define VLV_FIFO(plane, value) \
1990 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1992 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1993 struct intel_crtc *crtc)
1995 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1996 struct intel_uncore *uncore = &dev_priv->uncore;
1997 const struct intel_crtc_state *crtc_state =
1998 intel_atomic_get_new_crtc_state(state, crtc);
1999 const struct vlv_fifo_state *fifo_state =
2000 &crtc_state->wm.vlv.fifo_state;
2001 int sprite0_start, sprite1_start, fifo_size;
2002 u32 dsparb, dsparb2, dsparb3;
2004 if (!crtc_state->fifo_changed)
2007 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2008 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2009 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2011 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2012 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2014 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2017 * uncore.lock serves a double purpose here. It allows us to
2018 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2019 * it protects the DSPARB registers from getting clobbered by
2020 * parallel updates from multiple pipes.
2022 * intel_pipe_update_start() has already disabled interrupts
2023 * for us, so a plain spin_lock() is sufficient here.
2025 spin_lock(&uncore->lock);
2027 switch (crtc->pipe) {
2029 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2030 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2032 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2033 VLV_FIFO(SPRITEB, 0xff));
2034 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2035 VLV_FIFO(SPRITEB, sprite1_start));
2037 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2038 VLV_FIFO(SPRITEB_HI, 0x1));
2039 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2040 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2042 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2043 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2046 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2047 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2049 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2050 VLV_FIFO(SPRITED, 0xff));
2051 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2052 VLV_FIFO(SPRITED, sprite1_start));
2054 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2055 VLV_FIFO(SPRITED_HI, 0xff));
2056 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2057 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2059 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2060 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2063 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2064 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2066 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2067 VLV_FIFO(SPRITEF, 0xff));
2068 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2069 VLV_FIFO(SPRITEF, sprite1_start));
2071 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2072 VLV_FIFO(SPRITEF_HI, 0xff));
2073 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2074 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2076 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2077 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2083 intel_uncore_posting_read_fw(uncore, DSPARB);
2085 spin_unlock(&uncore->lock);
2090 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2092 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2093 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2094 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2095 struct intel_atomic_state *intel_state =
2096 to_intel_atomic_state(new_crtc_state->uapi.state);
2097 const struct intel_crtc_state *old_crtc_state =
2098 intel_atomic_get_old_crtc_state(intel_state, crtc);
2099 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2102 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2103 *intermediate = *optimal;
2105 intermediate->cxsr = false;
2109 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2110 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2111 !new_crtc_state->disable_cxsr;
2113 for (level = 0; level < intermediate->num_levels; level++) {
2114 enum plane_id plane_id;
2116 for_each_plane_id_on_crtc(crtc, plane_id) {
2117 intermediate->wm[level].plane[plane_id] =
2118 min(optimal->wm[level].plane[plane_id],
2119 active->wm[level].plane[plane_id]);
2122 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2123 active->sr[level].plane);
2124 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2125 active->sr[level].cursor);
2128 vlv_invalidate_wms(crtc, intermediate, level);
2132 * If our intermediate WM are identical to the final WM, then we can
2133 * omit the post-vblank programming; only update if it's different.
2135 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2136 new_crtc_state->wm.need_postvbl_update = true;
2141 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2142 struct vlv_wm_values *wm)
2144 struct intel_crtc *crtc;
2145 int num_active_pipes = 0;
2147 wm->level = dev_priv->wm.max_level;
2150 for_each_intel_crtc(&dev_priv->drm, crtc) {
2151 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2156 if (!wm_state->cxsr)
2160 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2163 if (num_active_pipes != 1)
2166 if (num_active_pipes > 1)
2167 wm->level = VLV_WM_LEVEL_PM2;
2169 for_each_intel_crtc(&dev_priv->drm, crtc) {
2170 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2171 enum pipe pipe = crtc->pipe;
2173 wm->pipe[pipe] = wm_state->wm[wm->level];
2174 if (crtc->active && wm->cxsr)
2175 wm->sr = wm_state->sr[wm->level];
2177 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2178 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2179 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2180 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2184 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2186 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2187 struct vlv_wm_values new_wm = {};
2189 vlv_merge_wm(dev_priv, &new_wm);
2191 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2194 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2195 chv_set_memory_dvfs(dev_priv, false);
2197 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2198 chv_set_memory_pm5(dev_priv, false);
2200 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2201 _intel_set_memory_cxsr(dev_priv, false);
2203 vlv_write_wm_values(dev_priv, &new_wm);
2205 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2206 _intel_set_memory_cxsr(dev_priv, true);
2208 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2209 chv_set_memory_pm5(dev_priv, true);
2211 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2212 chv_set_memory_dvfs(dev_priv, true);
2217 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2218 struct intel_crtc *crtc)
2220 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2221 const struct intel_crtc_state *crtc_state =
2222 intel_atomic_get_new_crtc_state(state, crtc);
2224 mutex_lock(&dev_priv->wm.wm_mutex);
2225 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2226 vlv_program_watermarks(dev_priv);
2227 mutex_unlock(&dev_priv->wm.wm_mutex);
2230 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2231 struct intel_crtc *crtc)
2233 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2234 const struct intel_crtc_state *crtc_state =
2235 intel_atomic_get_new_crtc_state(state, crtc);
2237 if (!crtc_state->wm.need_postvbl_update)
2240 mutex_lock(&dev_priv->wm.wm_mutex);
2241 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2242 vlv_program_watermarks(dev_priv);
2243 mutex_unlock(&dev_priv->wm.wm_mutex);
2246 static void i965_update_wm(struct intel_crtc *unused_crtc)
2248 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2249 struct intel_crtc *crtc;
2254 /* Calc sr entries for one plane configs */
2255 crtc = single_enabled_crtc(dev_priv);
2257 /* self-refresh has much higher latency */
2258 static const int sr_latency_ns = 12000;
2259 const struct drm_display_mode *adjusted_mode =
2260 &crtc->config->hw.adjusted_mode;
2261 const struct drm_framebuffer *fb =
2262 crtc->base.primary->state->fb;
2263 int clock = adjusted_mode->crtc_clock;
2264 int htotal = adjusted_mode->crtc_htotal;
2265 int hdisplay = crtc->config->pipe_src_w;
2266 int cpp = fb->format->cpp[0];
2269 entries = intel_wm_method2(clock, htotal,
2270 hdisplay, cpp, sr_latency_ns / 100);
2271 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2272 srwm = I965_FIFO_SIZE - entries;
2276 drm_dbg_kms(&dev_priv->drm,
2277 "self-refresh entries: %d, wm: %d\n",
2280 entries = intel_wm_method2(clock, htotal,
2281 crtc->base.cursor->state->crtc_w, 4,
2282 sr_latency_ns / 100);
2283 entries = DIV_ROUND_UP(entries,
2284 i965_cursor_wm_info.cacheline_size) +
2285 i965_cursor_wm_info.guard_size;
2287 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2288 if (cursor_sr > i965_cursor_wm_info.max_wm)
2289 cursor_sr = i965_cursor_wm_info.max_wm;
2291 drm_dbg_kms(&dev_priv->drm,
2292 "self-refresh watermark: display plane %d "
2293 "cursor %d\n", srwm, cursor_sr);
2295 cxsr_enabled = true;
2297 cxsr_enabled = false;
2298 /* Turn off self refresh if both pipes are enabled */
2299 intel_set_memory_cxsr(dev_priv, false);
2302 drm_dbg_kms(&dev_priv->drm,
2303 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2306 /* 965 has limitations... */
2307 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2311 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2312 FW_WM(8, PLANEC_OLD));
2313 /* update cursor SR watermark */
2314 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2317 intel_set_memory_cxsr(dev_priv, true);
2322 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2324 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2325 const struct intel_watermark_params *wm_info;
2330 int planea_wm, planeb_wm;
2331 struct intel_crtc *crtc, *enabled = NULL;
2333 if (IS_I945GM(dev_priv))
2334 wm_info = &i945_wm_info;
2335 else if (!IS_GEN(dev_priv, 2))
2336 wm_info = &i915_wm_info;
2338 wm_info = &i830_a_wm_info;
2340 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2341 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2342 if (intel_crtc_active(crtc)) {
2343 const struct drm_display_mode *adjusted_mode =
2344 &crtc->config->hw.adjusted_mode;
2345 const struct drm_framebuffer *fb =
2346 crtc->base.primary->state->fb;
2349 if (IS_GEN(dev_priv, 2))
2352 cpp = fb->format->cpp[0];
2354 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2355 wm_info, fifo_size, cpp,
2356 pessimal_latency_ns);
2359 planea_wm = fifo_size - wm_info->guard_size;
2360 if (planea_wm > (long)wm_info->max_wm)
2361 planea_wm = wm_info->max_wm;
2364 if (IS_GEN(dev_priv, 2))
2365 wm_info = &i830_bc_wm_info;
2367 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2368 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2369 if (intel_crtc_active(crtc)) {
2370 const struct drm_display_mode *adjusted_mode =
2371 &crtc->config->hw.adjusted_mode;
2372 const struct drm_framebuffer *fb =
2373 crtc->base.primary->state->fb;
2376 if (IS_GEN(dev_priv, 2))
2379 cpp = fb->format->cpp[0];
2381 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2382 wm_info, fifo_size, cpp,
2383 pessimal_latency_ns);
2384 if (enabled == NULL)
2389 planeb_wm = fifo_size - wm_info->guard_size;
2390 if (planeb_wm > (long)wm_info->max_wm)
2391 planeb_wm = wm_info->max_wm;
2394 drm_dbg_kms(&dev_priv->drm,
2395 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2397 if (IS_I915GM(dev_priv) && enabled) {
2398 struct drm_i915_gem_object *obj;
2400 obj = intel_fb_obj(enabled->base.primary->state->fb);
2402 /* self-refresh seems busted with untiled */
2403 if (!i915_gem_object_is_tiled(obj))
2408 * Overlay gets an aggressive default since video jitter is bad.
2412 /* Play safe and disable self-refresh before adjusting watermarks. */
2413 intel_set_memory_cxsr(dev_priv, false);
2415 /* Calc sr entries for one plane configs */
2416 if (HAS_FW_BLC(dev_priv) && enabled) {
2417 /* self-refresh has much higher latency */
2418 static const int sr_latency_ns = 6000;
2419 const struct drm_display_mode *adjusted_mode =
2420 &enabled->config->hw.adjusted_mode;
2421 const struct drm_framebuffer *fb =
2422 enabled->base.primary->state->fb;
2423 int clock = adjusted_mode->crtc_clock;
2424 int htotal = adjusted_mode->crtc_htotal;
2425 int hdisplay = enabled->config->pipe_src_w;
2429 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2432 cpp = fb->format->cpp[0];
2434 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2435 sr_latency_ns / 100);
2436 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2437 drm_dbg_kms(&dev_priv->drm,
2438 "self-refresh entries: %d\n", entries);
2439 srwm = wm_info->fifo_size - entries;
2443 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2444 I915_WRITE(FW_BLC_SELF,
2445 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2447 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2450 drm_dbg_kms(&dev_priv->drm,
2451 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2452 planea_wm, planeb_wm, cwm, srwm);
2454 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2455 fwater_hi = (cwm & 0x1f);
2457 /* Set request length to 8 cachelines per fetch */
2458 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2459 fwater_hi = fwater_hi | (1 << 8);
2461 I915_WRITE(FW_BLC, fwater_lo);
2462 I915_WRITE(FW_BLC2, fwater_hi);
2465 intel_set_memory_cxsr(dev_priv, true);
2468 static void i845_update_wm(struct intel_crtc *unused_crtc)
2470 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2471 struct intel_crtc *crtc;
2472 const struct drm_display_mode *adjusted_mode;
2476 crtc = single_enabled_crtc(dev_priv);
2480 adjusted_mode = &crtc->config->hw.adjusted_mode;
2481 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2483 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2484 4, pessimal_latency_ns);
2485 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2486 fwater_lo |= (3<<8) | planea_wm;
2488 drm_dbg_kms(&dev_priv->drm,
2489 "Setting FIFO watermarks - A: %d\n", planea_wm);
2491 I915_WRITE(FW_BLC, fwater_lo);
2494 /* latency must be in 0.1us units. */
2495 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2497 unsigned int latency)
2501 ret = intel_wm_method1(pixel_rate, cpp, latency);
2502 ret = DIV_ROUND_UP(ret, 64) + 2;
2507 /* latency must be in 0.1us units. */
2508 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2509 unsigned int htotal,
2512 unsigned int latency)
2516 ret = intel_wm_method2(pixel_rate, htotal,
2517 width, cpp, latency);
2518 ret = DIV_ROUND_UP(ret, 64) + 2;
2523 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2526 * Neither of these should be possible since this function shouldn't be
2527 * called if the CRTC is off or the plane is invisible. But let's be
2528 * extra paranoid to avoid a potential divide-by-zero if we screw up
2529 * elsewhere in the driver.
2533 if (WARN_ON(!horiz_pixels))
2536 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2539 struct ilk_wm_maximums {
2547 * For both WM_PIPE and WM_LP.
2548 * mem_value must be in 0.1us units.
2550 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2551 const struct intel_plane_state *plane_state,
2552 u32 mem_value, bool is_lp)
2554 u32 method1, method2;
2560 if (!intel_wm_plane_visible(crtc_state, plane_state))
2563 cpp = plane_state->hw.fb->format->cpp[0];
2565 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2570 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2571 crtc_state->hw.adjusted_mode.crtc_htotal,
2572 drm_rect_width(&plane_state->uapi.dst),
2575 return min(method1, method2);
2579 * For both WM_PIPE and WM_LP.
2580 * mem_value must be in 0.1us units.
2582 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2583 const struct intel_plane_state *plane_state,
2586 u32 method1, method2;
2592 if (!intel_wm_plane_visible(crtc_state, plane_state))
2595 cpp = plane_state->hw.fb->format->cpp[0];
2597 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2598 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2599 crtc_state->hw.adjusted_mode.crtc_htotal,
2600 drm_rect_width(&plane_state->uapi.dst),
2602 return min(method1, method2);
2606 * For both WM_PIPE and WM_LP.
2607 * mem_value must be in 0.1us units.
2609 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2610 const struct intel_plane_state *plane_state,
2618 if (!intel_wm_plane_visible(crtc_state, plane_state))
2621 cpp = plane_state->hw.fb->format->cpp[0];
2623 return ilk_wm_method2(crtc_state->pixel_rate,
2624 crtc_state->hw.adjusted_mode.crtc_htotal,
2625 drm_rect_width(&plane_state->uapi.dst),
2629 /* Only for WM_LP. */
2630 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2631 const struct intel_plane_state *plane_state,
2636 if (!intel_wm_plane_visible(crtc_state, plane_state))
2639 cpp = plane_state->hw.fb->format->cpp[0];
2641 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2646 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2648 if (INTEL_GEN(dev_priv) >= 8)
2650 else if (INTEL_GEN(dev_priv) >= 7)
2657 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2658 int level, bool is_sprite)
2660 if (INTEL_GEN(dev_priv) >= 8)
2661 /* BDW primary/sprite plane watermarks */
2662 return level == 0 ? 255 : 2047;
2663 else if (INTEL_GEN(dev_priv) >= 7)
2664 /* IVB/HSW primary/sprite plane watermarks */
2665 return level == 0 ? 127 : 1023;
2666 else if (!is_sprite)
2667 /* ILK/SNB primary plane watermarks */
2668 return level == 0 ? 127 : 511;
2670 /* ILK/SNB sprite plane watermarks */
2671 return level == 0 ? 63 : 255;
2675 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2677 if (INTEL_GEN(dev_priv) >= 7)
2678 return level == 0 ? 63 : 255;
2680 return level == 0 ? 31 : 63;
2683 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2685 if (INTEL_GEN(dev_priv) >= 8)
2691 /* Calculate the maximum primary/sprite plane watermark */
2692 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2694 const struct intel_wm_config *config,
2695 enum intel_ddb_partitioning ddb_partitioning,
2698 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2700 /* if sprites aren't enabled, sprites get nothing */
2701 if (is_sprite && !config->sprites_enabled)
2704 /* HSW allows LP1+ watermarks even with multiple pipes */
2705 if (level == 0 || config->num_pipes_active > 1) {
2706 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2709 * For some reason the non self refresh
2710 * FIFO size is only half of the self
2711 * refresh FIFO size on ILK/SNB.
2713 if (INTEL_GEN(dev_priv) <= 6)
2717 if (config->sprites_enabled) {
2718 /* level 0 is always calculated with 1:1 split */
2719 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2728 /* clamp to max that the registers can hold */
2729 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2732 /* Calculate the maximum cursor plane watermark */
2733 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2735 const struct intel_wm_config *config)
2737 /* HSW LP1+ watermarks w/ multiple pipes */
2738 if (level > 0 && config->num_pipes_active > 1)
2741 /* otherwise just report max that registers can hold */
2742 return ilk_cursor_wm_reg_max(dev_priv, level);
2745 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2747 const struct intel_wm_config *config,
2748 enum intel_ddb_partitioning ddb_partitioning,
2749 struct ilk_wm_maximums *max)
2751 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2752 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2753 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2754 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2757 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2759 struct ilk_wm_maximums *max)
2761 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2762 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2763 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2764 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2767 static bool ilk_validate_wm_level(int level,
2768 const struct ilk_wm_maximums *max,
2769 struct intel_wm_level *result)
2773 /* already determined to be invalid? */
2774 if (!result->enable)
2777 result->enable = result->pri_val <= max->pri &&
2778 result->spr_val <= max->spr &&
2779 result->cur_val <= max->cur;
2781 ret = result->enable;
2784 * HACK until we can pre-compute everything,
2785 * and thus fail gracefully if LP0 watermarks
2788 if (level == 0 && !result->enable) {
2789 if (result->pri_val > max->pri)
2790 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2791 level, result->pri_val, max->pri);
2792 if (result->spr_val > max->spr)
2793 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2794 level, result->spr_val, max->spr);
2795 if (result->cur_val > max->cur)
2796 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2797 level, result->cur_val, max->cur);
2799 result->pri_val = min_t(u32, result->pri_val, max->pri);
2800 result->spr_val = min_t(u32, result->spr_val, max->spr);
2801 result->cur_val = min_t(u32, result->cur_val, max->cur);
2802 result->enable = true;
2808 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2809 const struct intel_crtc *crtc,
2811 struct intel_crtc_state *crtc_state,
2812 const struct intel_plane_state *pristate,
2813 const struct intel_plane_state *sprstate,
2814 const struct intel_plane_state *curstate,
2815 struct intel_wm_level *result)
2817 u16 pri_latency = dev_priv->wm.pri_latency[level];
2818 u16 spr_latency = dev_priv->wm.spr_latency[level];
2819 u16 cur_latency = dev_priv->wm.cur_latency[level];
2821 /* WM1+ latency values stored in 0.5us units */
2829 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2830 pri_latency, level);
2831 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2835 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2838 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2840 result->enable = true;
2843 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2846 struct intel_uncore *uncore = &dev_priv->uncore;
2848 if (INTEL_GEN(dev_priv) >= 9) {
2851 int level, max_level = ilk_wm_max_level(dev_priv);
2853 /* read the first set of memory latencies[0:3] */
2854 val = 0; /* data0 to be programmed to 0 for first set */
2855 ret = sandybridge_pcode_read(dev_priv,
2856 GEN9_PCODE_READ_MEM_LATENCY,
2860 drm_err(&dev_priv->drm,
2861 "SKL Mailbox read error = %d\n", ret);
2865 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2866 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2867 GEN9_MEM_LATENCY_LEVEL_MASK;
2868 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2869 GEN9_MEM_LATENCY_LEVEL_MASK;
2870 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2871 GEN9_MEM_LATENCY_LEVEL_MASK;
2873 /* read the second set of memory latencies[4:7] */
2874 val = 1; /* data0 to be programmed to 1 for second set */
2875 ret = sandybridge_pcode_read(dev_priv,
2876 GEN9_PCODE_READ_MEM_LATENCY,
2879 drm_err(&dev_priv->drm,
2880 "SKL Mailbox read error = %d\n", ret);
2884 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2885 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2886 GEN9_MEM_LATENCY_LEVEL_MASK;
2887 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2888 GEN9_MEM_LATENCY_LEVEL_MASK;
2889 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2890 GEN9_MEM_LATENCY_LEVEL_MASK;
2893 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2894 * need to be disabled. We make sure to sanitize the values out
2895 * of the punit to satisfy this requirement.
2897 for (level = 1; level <= max_level; level++) {
2898 if (wm[level] == 0) {
2899 for (i = level + 1; i <= max_level; i++)
2906 * WaWmMemoryReadLatency:skl+,glk
2908 * punit doesn't take into account the read latency so we need
2909 * to add 2us to the various latency levels we retrieve from the
2910 * punit when level 0 response data us 0us.
2914 for (level = 1; level <= max_level; level++) {
2922 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2923 * If we could not get dimm info enable this WA to prevent from
2924 * any underrun. If not able to get Dimm info assume 16GB dimm
2925 * to avoid any underrun.
2927 if (dev_priv->dram_info.is_16gb_dimm)
2930 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2931 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2933 wm[0] = (sskpd >> 56) & 0xFF;
2935 wm[0] = sskpd & 0xF;
2936 wm[1] = (sskpd >> 4) & 0xFF;
2937 wm[2] = (sskpd >> 12) & 0xFF;
2938 wm[3] = (sskpd >> 20) & 0x1FF;
2939 wm[4] = (sskpd >> 32) & 0x1FF;
2940 } else if (INTEL_GEN(dev_priv) >= 6) {
2941 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2943 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2944 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2945 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2946 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2947 } else if (INTEL_GEN(dev_priv) >= 5) {
2948 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2950 /* ILK primary LP0 latency is 700 ns */
2952 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2953 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2955 MISSING_CASE(INTEL_DEVID(dev_priv));
2959 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2962 /* ILK sprite LP0 latency is 1300 ns */
2963 if (IS_GEN(dev_priv, 5))
2967 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2970 /* ILK cursor LP0 latency is 1300 ns */
2971 if (IS_GEN(dev_priv, 5))
2975 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2977 /* how many WM levels are we expecting */
2978 if (INTEL_GEN(dev_priv) >= 9)
2980 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2982 else if (INTEL_GEN(dev_priv) >= 6)
2988 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2992 int level, max_level = ilk_wm_max_level(dev_priv);
2994 for (level = 0; level <= max_level; level++) {
2995 unsigned int latency = wm[level];
2998 drm_dbg_kms(&dev_priv->drm,
2999 "%s WM%d latency not provided\n",
3005 * - latencies are in us on gen9.
3006 * - before then, WM1+ latency values are in 0.5us units
3008 if (INTEL_GEN(dev_priv) >= 9)
3013 drm_dbg_kms(&dev_priv->drm,
3014 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3015 wm[level], latency / 10, latency % 10);
3019 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3022 int level, max_level = ilk_wm_max_level(dev_priv);
3027 wm[0] = max(wm[0], min);
3028 for (level = 1; level <= max_level; level++)
3029 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3034 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3039 * The BIOS provided WM memory latency values are often
3040 * inadequate for high resolution displays. Adjust them.
3042 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3043 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3044 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3049 drm_dbg_kms(&dev_priv->drm,
3050 "WM latency values increased to avoid potential underruns\n");
3051 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3052 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3053 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3056 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3059 * On some SNB machines (Thinkpad X220 Tablet at least)
3060 * LP3 usage can cause vblank interrupts to be lost.
3061 * The DEIIR bit will go high but it looks like the CPU
3062 * never gets interrupted.
3064 * It's not clear whether other interrupt source could
3065 * be affected or if this is somehow limited to vblank
3066 * interrupts only. To play it safe we disable LP3
3067 * watermarks entirely.
3069 if (dev_priv->wm.pri_latency[3] == 0 &&
3070 dev_priv->wm.spr_latency[3] == 0 &&
3071 dev_priv->wm.cur_latency[3] == 0)
3074 dev_priv->wm.pri_latency[3] = 0;
3075 dev_priv->wm.spr_latency[3] = 0;
3076 dev_priv->wm.cur_latency[3] = 0;
3078 drm_dbg_kms(&dev_priv->drm,
3079 "LP3 watermarks disabled due to potential for lost interrupts\n");
3080 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3081 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3082 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3085 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3087 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3089 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3090 sizeof(dev_priv->wm.pri_latency));
3091 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3092 sizeof(dev_priv->wm.pri_latency));
3094 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3095 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3097 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3098 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3099 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3101 if (IS_GEN(dev_priv, 6)) {
3102 snb_wm_latency_quirk(dev_priv);
3103 snb_wm_lp3_irq_quirk(dev_priv);
3107 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3109 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3110 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3113 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3114 struct intel_pipe_wm *pipe_wm)
3116 /* LP0 watermark maximums depend on this pipe alone */
3117 const struct intel_wm_config config = {
3118 .num_pipes_active = 1,
3119 .sprites_enabled = pipe_wm->sprites_enabled,
3120 .sprites_scaled = pipe_wm->sprites_scaled,
3122 struct ilk_wm_maximums max;
3124 /* LP0 watermarks always use 1/2 DDB partitioning */
3125 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3127 /* At least LP0 must be valid */
3128 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3129 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3136 /* Compute new watermarks for the pipe */
3137 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3139 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3140 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3141 struct intel_pipe_wm *pipe_wm;
3142 struct intel_plane *plane;
3143 const struct intel_plane_state *plane_state;
3144 const struct intel_plane_state *pristate = NULL;
3145 const struct intel_plane_state *sprstate = NULL;
3146 const struct intel_plane_state *curstate = NULL;
3147 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3148 struct ilk_wm_maximums max;
3150 pipe_wm = &crtc_state->wm.ilk.optimal;
3152 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3153 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3154 pristate = plane_state;
3155 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3156 sprstate = plane_state;
3157 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3158 curstate = plane_state;
3161 pipe_wm->pipe_enabled = crtc_state->hw.active;
3163 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3164 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3165 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3166 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3169 usable_level = max_level;
3171 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3172 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3175 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3176 if (pipe_wm->sprites_scaled)
3179 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3180 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3181 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3183 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3186 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3188 for (level = 1; level <= usable_level; level++) {
3189 struct intel_wm_level *wm = &pipe_wm->wm[level];
3191 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3192 pristate, sprstate, curstate, wm);
3195 * Disable any watermark level that exceeds the
3196 * register maximums since such watermarks are
3199 if (!ilk_validate_wm_level(level, &max, wm)) {
3200 memset(wm, 0, sizeof(*wm));
3209 * Build a set of 'intermediate' watermark values that satisfy both the old
3210 * state and the new state. These can be programmed to the hardware
3213 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3215 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3216 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3217 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3218 struct intel_atomic_state *intel_state =
3219 to_intel_atomic_state(newstate->uapi.state);
3220 const struct intel_crtc_state *oldstate =
3221 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3222 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3223 int level, max_level = ilk_wm_max_level(dev_priv);
3226 * Start with the final, target watermarks, then combine with the
3227 * currently active watermarks to get values that are safe both before
3228 * and after the vblank.
3230 *a = newstate->wm.ilk.optimal;
3231 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3232 intel_state->skip_intermediate_wm)
3235 a->pipe_enabled |= b->pipe_enabled;
3236 a->sprites_enabled |= b->sprites_enabled;
3237 a->sprites_scaled |= b->sprites_scaled;
3239 for (level = 0; level <= max_level; level++) {
3240 struct intel_wm_level *a_wm = &a->wm[level];
3241 const struct intel_wm_level *b_wm = &b->wm[level];
3243 a_wm->enable &= b_wm->enable;
3244 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3245 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3246 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3247 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3251 * We need to make sure that these merged watermark values are
3252 * actually a valid configuration themselves. If they're not,
3253 * there's no safe way to transition from the old state to
3254 * the new state, so we need to fail the atomic transaction.
3256 if (!ilk_validate_pipe_wm(dev_priv, a))
3260 * If our intermediate WM are identical to the final WM, then we can
3261 * omit the post-vblank programming; only update if it's different.
3263 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3264 newstate->wm.need_postvbl_update = true;
3270 * Merge the watermarks from all active pipes for a specific level.
3272 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3274 struct intel_wm_level *ret_wm)
3276 const struct intel_crtc *intel_crtc;
3278 ret_wm->enable = true;
3280 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3281 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3282 const struct intel_wm_level *wm = &active->wm[level];
3284 if (!active->pipe_enabled)
3288 * The watermark values may have been used in the past,
3289 * so we must maintain them in the registers for some
3290 * time even if the level is now disabled.
3293 ret_wm->enable = false;
3295 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3296 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3297 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3298 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3303 * Merge all low power watermarks for all active pipes.
3305 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3306 const struct intel_wm_config *config,
3307 const struct ilk_wm_maximums *max,
3308 struct intel_pipe_wm *merged)
3310 int level, max_level = ilk_wm_max_level(dev_priv);
3311 int last_enabled_level = max_level;
3313 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3314 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3315 config->num_pipes_active > 1)
3316 last_enabled_level = 0;
3318 /* ILK: FBC WM must be disabled always */
3319 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3321 /* merge each WM1+ level */
3322 for (level = 1; level <= max_level; level++) {
3323 struct intel_wm_level *wm = &merged->wm[level];
3325 ilk_merge_wm_level(dev_priv, level, wm);
3327 if (level > last_enabled_level)
3329 else if (!ilk_validate_wm_level(level, max, wm))
3330 /* make sure all following levels get disabled */
3331 last_enabled_level = level - 1;
3334 * The spec says it is preferred to disable
3335 * FBC WMs instead of disabling a WM level.
3337 if (wm->fbc_val > max->fbc) {
3339 merged->fbc_wm_enabled = false;
3344 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3346 * FIXME this is racy. FBC might get enabled later.
3347 * What we should check here is whether FBC can be
3348 * enabled sometime later.
3350 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3351 intel_fbc_is_active(dev_priv)) {
3352 for (level = 2; level <= max_level; level++) {
3353 struct intel_wm_level *wm = &merged->wm[level];
3360 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3362 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3363 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3366 /* The value we need to program into the WM_LPx latency field */
3367 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3370 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3373 return dev_priv->wm.pri_latency[level];
3376 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3377 const struct intel_pipe_wm *merged,
3378 enum intel_ddb_partitioning partitioning,
3379 struct ilk_wm_values *results)
3381 struct intel_crtc *intel_crtc;
3384 results->enable_fbc_wm = merged->fbc_wm_enabled;
3385 results->partitioning = partitioning;
3387 /* LP1+ register values */
3388 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3389 const struct intel_wm_level *r;
3391 level = ilk_wm_lp_to_level(wm_lp, merged);
3393 r = &merged->wm[level];
3396 * Maintain the watermark values even if the level is
3397 * disabled. Doing otherwise could cause underruns.
3399 results->wm_lp[wm_lp - 1] =
3400 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3401 (r->pri_val << WM1_LP_SR_SHIFT) |
3405 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3407 if (INTEL_GEN(dev_priv) >= 8)
3408 results->wm_lp[wm_lp - 1] |=
3409 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3411 results->wm_lp[wm_lp - 1] |=
3412 r->fbc_val << WM1_LP_FBC_SHIFT;
3415 * Always set WM1S_LP_EN when spr_val != 0, even if the
3416 * level is disabled. Doing otherwise could cause underruns.
3418 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3419 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3420 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3422 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3425 /* LP0 register values */
3426 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3427 enum pipe pipe = intel_crtc->pipe;
3428 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3429 const struct intel_wm_level *r = &pipe_wm->wm[0];
3431 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3434 results->wm_pipe[pipe] =
3435 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3436 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3441 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3442 * case both are at the same level. Prefer r1 in case they're the same. */
3443 static struct intel_pipe_wm *
3444 ilk_find_best_result(struct drm_i915_private *dev_priv,
3445 struct intel_pipe_wm *r1,
3446 struct intel_pipe_wm *r2)
3448 int level, max_level = ilk_wm_max_level(dev_priv);
3449 int level1 = 0, level2 = 0;
3451 for (level = 1; level <= max_level; level++) {
3452 if (r1->wm[level].enable)
3454 if (r2->wm[level].enable)
3458 if (level1 == level2) {
3459 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3463 } else if (level1 > level2) {
3470 /* dirty bits used to track which watermarks need changes */
3471 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3472 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3473 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3474 #define WM_DIRTY_FBC (1 << 24)
3475 #define WM_DIRTY_DDB (1 << 25)
3477 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3478 const struct ilk_wm_values *old,
3479 const struct ilk_wm_values *new)
3481 unsigned int dirty = 0;
3485 for_each_pipe(dev_priv, pipe) {
3486 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3487 dirty |= WM_DIRTY_PIPE(pipe);
3488 /* Must disable LP1+ watermarks too */
3489 dirty |= WM_DIRTY_LP_ALL;
3493 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3494 dirty |= WM_DIRTY_FBC;
3495 /* Must disable LP1+ watermarks too */
3496 dirty |= WM_DIRTY_LP_ALL;
3499 if (old->partitioning != new->partitioning) {
3500 dirty |= WM_DIRTY_DDB;
3501 /* Must disable LP1+ watermarks too */
3502 dirty |= WM_DIRTY_LP_ALL;
3505 /* LP1+ watermarks already deemed dirty, no need to continue */
3506 if (dirty & WM_DIRTY_LP_ALL)
3509 /* Find the lowest numbered LP1+ watermark in need of an update... */
3510 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3511 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3512 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3516 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3517 for (; wm_lp <= 3; wm_lp++)
3518 dirty |= WM_DIRTY_LP(wm_lp);
3523 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3526 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3527 bool changed = false;
3529 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3530 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3531 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3534 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3535 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3536 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3539 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3540 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3541 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3546 * Don't touch WM1S_LP_EN here.
3547 * Doing so could cause underruns.
3554 * The spec says we shouldn't write when we don't need, because every write
3555 * causes WMs to be re-evaluated, expending some power.
3557 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3558 struct ilk_wm_values *results)
3560 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3564 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3568 _ilk_disable_lp_wm(dev_priv, dirty);
3570 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3571 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3572 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3573 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3574 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3575 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3577 if (dirty & WM_DIRTY_DDB) {
3578 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3579 val = I915_READ(WM_MISC);
3580 if (results->partitioning == INTEL_DDB_PART_1_2)
3581 val &= ~WM_MISC_DATA_PARTITION_5_6;
3583 val |= WM_MISC_DATA_PARTITION_5_6;
3584 I915_WRITE(WM_MISC, val);
3586 val = I915_READ(DISP_ARB_CTL2);
3587 if (results->partitioning == INTEL_DDB_PART_1_2)
3588 val &= ~DISP_DATA_PARTITION_5_6;
3590 val |= DISP_DATA_PARTITION_5_6;
3591 I915_WRITE(DISP_ARB_CTL2, val);
3595 if (dirty & WM_DIRTY_FBC) {
3596 val = I915_READ(DISP_ARB_CTL);
3597 if (results->enable_fbc_wm)
3598 val &= ~DISP_FBC_WM_DIS;
3600 val |= DISP_FBC_WM_DIS;
3601 I915_WRITE(DISP_ARB_CTL, val);
3604 if (dirty & WM_DIRTY_LP(1) &&
3605 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3606 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3608 if (INTEL_GEN(dev_priv) >= 7) {
3609 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3610 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3611 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3612 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3615 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3616 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3617 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3618 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3619 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3620 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3622 dev_priv->wm.hw = *results;
3625 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3627 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3630 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3633 int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3634 u8 enabled_slices_mask = 0;
3636 for (i = 0; i < max_slices; i++) {
3637 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3638 enabled_slices_mask |= BIT(i);
3641 return enabled_slices_mask;
3645 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3646 * so assume we'll always need it in order to avoid underruns.
3648 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3650 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3654 intel_has_sagv(struct drm_i915_private *dev_priv)
3656 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3657 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3661 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3663 if (INTEL_GEN(dev_priv) >= 12) {
3667 ret = sandybridge_pcode_read(dev_priv,
3668 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3671 dev_priv->sagv_block_time_us = val;
3675 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3676 } else if (IS_GEN(dev_priv, 11)) {
3677 dev_priv->sagv_block_time_us = 10;
3679 } else if (IS_GEN(dev_priv, 10)) {
3680 dev_priv->sagv_block_time_us = 20;
3682 } else if (IS_GEN(dev_priv, 9)) {
3683 dev_priv->sagv_block_time_us = 30;
3686 MISSING_CASE(INTEL_GEN(dev_priv));
3689 /* Default to an unusable block time */
3690 dev_priv->sagv_block_time_us = -1;
3694 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3695 * depending on power and performance requirements. The display engine access
3696 * to system memory is blocked during the adjustment time. Because of the
3697 * blocking time, having this enabled can cause full system hangs and/or pipe
3698 * underruns if we don't meet all of the following requirements:
3700 * - <= 1 pipe enabled
3701 * - All planes can enable watermarks for latencies >= SAGV engine block time
3702 * - We're not using an interlaced display configuration
3705 intel_enable_sagv(struct drm_i915_private *dev_priv)
3709 if (!intel_has_sagv(dev_priv))
3712 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3715 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3716 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3719 /* We don't need to wait for SAGV when enabling */
3722 * Some skl systems, pre-release machines in particular,
3723 * don't actually have SAGV.
3725 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3726 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3727 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3729 } else if (ret < 0) {
3730 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3734 dev_priv->sagv_status = I915_SAGV_ENABLED;
3739 intel_disable_sagv(struct drm_i915_private *dev_priv)
3743 if (!intel_has_sagv(dev_priv))
3746 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3749 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3750 /* bspec says to keep retrying for at least 1 ms */
3751 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3753 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3756 * Some skl systems, pre-release machines in particular,
3757 * don't actually have SAGV.
3759 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3760 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3761 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3763 } else if (ret < 0) {
3764 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3768 dev_priv->sagv_status = I915_SAGV_DISABLED;
3772 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3774 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3775 const struct intel_bw_state *new_bw_state;
3776 const struct intel_bw_state *old_bw_state;
3780 * Just return if we can't control SAGV or don't have it.
3781 * This is different from situation when we have SAGV but just can't
3782 * afford it due to DBuf limitation - in case if SAGV is completely
3783 * disabled in a BIOS, we are not even allowed to send a PCode request,
3784 * as it will throw an error. So have to check it here.
3786 if (!intel_has_sagv(dev_priv))
3789 new_bw_state = intel_atomic_get_new_bw_state(state);
3793 if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3794 intel_disable_sagv(dev_priv);
3798 old_bw_state = intel_atomic_get_old_bw_state(state);
3802 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3805 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3808 * If new mask is zero - means there is nothing to mask,
3809 * we can only unmask, which should be done in unmask.
3815 * Restrict required qgv points before updating the configuration.
3816 * According to BSpec we can't mask and unmask qgv points at the same
3817 * time. Also masking should be done before updating the configuration
3818 * and unmasking afterwards.
3820 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3823 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3825 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3826 const struct intel_bw_state *new_bw_state;
3827 const struct intel_bw_state *old_bw_state;
3831 * Just return if we can't control SAGV or don't have it.
3832 * This is different from situation when we have SAGV but just can't
3833 * afford it due to DBuf limitation - in case if SAGV is completely
3834 * disabled in a BIOS, we are not even allowed to send a PCode request,
3835 * as it will throw an error. So have to check it here.
3837 if (!intel_has_sagv(dev_priv))
3840 new_bw_state = intel_atomic_get_new_bw_state(state);
3844 if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3845 intel_enable_sagv(dev_priv);
3849 old_bw_state = intel_atomic_get_old_bw_state(state);
3853 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3856 new_mask = new_bw_state->qgv_points_mask;
3859 * Allow required qgv points after updating the configuration.
3860 * According to BSpec we can't mask and unmask qgv points at the same
3861 * time. Also masking should be done before updating the configuration
3862 * and unmasking afterwards.
3864 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3867 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3869 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3870 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3871 struct intel_plane *plane;
3872 const struct intel_plane_state *plane_state;
3875 if (!intel_has_sagv(dev_priv))
3878 if (!crtc_state->hw.active)
3881 if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3884 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3885 const struct skl_plane_wm *wm =
3886 &crtc_state->wm.skl.optimal.planes[plane->id];
3888 /* Skip this plane if it's not enabled */
3889 if (!wm->wm[0].plane_en)
3892 /* Find the highest enabled wm level for this plane */
3893 for (level = ilk_wm_max_level(dev_priv);
3894 !wm->wm[level].plane_en; --level)
3897 latency = dev_priv->wm.skl_latency[level];
3899 if (skl_needs_memory_bw_wa(dev_priv) &&
3900 plane_state->uapi.fb->modifier ==
3901 I915_FORMAT_MOD_X_TILED)
3905 * If any of the planes on this pipe don't enable wm levels that
3906 * incur memory latencies higher than sagv_block_time_us we
3907 * can't enable SAGV.
3909 if (latency < dev_priv->sagv_block_time_us)
3916 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3918 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3919 enum plane_id plane_id;
3921 if (!crtc_state->hw.active)
3924 for_each_plane_id_on_crtc(crtc, plane_id) {
3925 const struct skl_ddb_entry *plane_alloc =
3926 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3927 const struct skl_plane_wm *wm =
3928 &crtc_state->wm.skl.optimal.planes[plane_id];
3930 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3937 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3939 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3940 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3942 if (INTEL_GEN(dev_priv) >= 12)
3943 return tgl_crtc_can_enable_sagv(crtc_state);
3945 return skl_crtc_can_enable_sagv(crtc_state);
3948 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3949 const struct intel_bw_state *bw_state)
3951 if (INTEL_GEN(dev_priv) < 11 &&
3952 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3955 return bw_state->pipe_sagv_reject == 0;
3958 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3960 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3962 struct intel_crtc *crtc;
3963 struct intel_crtc_state *new_crtc_state;
3964 struct intel_bw_state *new_bw_state = NULL;
3965 const struct intel_bw_state *old_bw_state = NULL;
3968 for_each_new_intel_crtc_in_state(state, crtc,
3969 new_crtc_state, i) {
3970 new_bw_state = intel_atomic_get_bw_state(state);
3971 if (IS_ERR(new_bw_state))
3972 return PTR_ERR(new_bw_state);
3974 old_bw_state = intel_atomic_get_old_bw_state(state);
3976 if (intel_crtc_can_enable_sagv(new_crtc_state))
3977 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3979 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3985 new_bw_state->active_pipes =
3986 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3988 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3989 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3994 for_each_new_intel_crtc_in_state(state, crtc,
3995 new_crtc_state, i) {
3996 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
3999 * We store use_sagv_wm in the crtc state rather than relying on
4000 * that bw state since we have no convenient way to get at the
4001 * latter from the plane commit hooks (especially in the legacy
4004 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4005 intel_can_enable_sagv(dev_priv, new_bw_state);
4008 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4009 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4010 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4013 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4014 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4023 * Calculate initial DBuf slice offset, based on slice size
4024 * and mask(i.e if slice size is 1024 and second slice is enabled
4025 * offset would be 1024)
4028 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4032 unsigned int offset = 0;
4034 if (!dbuf_slice_mask)
4037 offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4039 WARN_ON(offset >= ddb_size);
4043 u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4045 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4046 drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4048 if (INTEL_GEN(dev_priv) < 11)
4049 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4054 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4055 const struct skl_ddb_entry *entry)
4058 u16 ddb_size = intel_get_ddb_size(dev_priv);
4059 u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4060 u16 slice_size = ddb_size / num_supported_slices;
4064 if (!skl_ddb_entry_size(entry))
4067 start_slice = entry->start / slice_size;
4068 end_slice = (entry->end - 1) / slice_size;
4071 * Per plane DDB entry can in a really worst case be on multiple slices
4072 * but single entry is anyway contigious.
4074 while (start_slice <= end_slice) {
4075 slice_mask |= BIT(start_slice);
4082 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4086 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4087 const struct intel_crtc_state *crtc_state,
4088 const u64 total_data_rate,
4089 struct skl_ddb_entry *alloc, /* out */
4090 int *num_active /* out */)
4092 struct drm_atomic_state *state = crtc_state->uapi.state;
4093 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4094 struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4095 const struct intel_crtc *crtc;
4096 u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4097 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4098 struct intel_dbuf_state *new_dbuf_state =
4099 intel_atomic_get_new_dbuf_state(intel_state);
4100 const struct intel_dbuf_state *old_dbuf_state =
4101 intel_atomic_get_old_dbuf_state(intel_state);
4102 u8 active_pipes = new_dbuf_state->active_pipes;
4106 u32 dbuf_slice_mask;
4109 u32 total_slice_mask;
4113 *num_active = hweight8(active_pipes);
4115 if (!crtc_state->hw.active) {
4121 ddb_size = intel_get_ddb_size(dev_priv);
4123 slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4126 * If the state doesn't change the active CRTC's or there is no
4127 * modeset request, then there's no need to recalculate;
4128 * the existing pipe allocation limits should remain unchanged.
4129 * Note that we're safe from racing commits since any racing commit
4130 * that changes the active CRTC list or do modeset would need to
4131 * grab _all_ crtc locks, including the one we currently hold.
4133 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes &&
4134 !dev_priv->wm.distrust_bios_wm) {
4136 * alloc may be cleared by clear_intel_crtc_state,
4137 * copy from old state to be sure
4139 * FIXME get rid of this mess
4141 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4146 * Get allowed DBuf slices for correspondent pipe and platform.
4148 dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4151 * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4152 * and slice size is 1024, the offset would be 1024
4154 offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4155 slice_size, ddb_size);
4158 * Figure out total size of allowed DBuf slices, which is basically
4159 * a number of allowed slices for that pipe multiplied by slice size.
4161 * range ddb entries are still allocated in proportion to display width.
4163 ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4166 * Watermark/ddb requirement highly depends upon width of the
4167 * framebuffer, So instead of allocating DDB equally among pipes
4168 * distribute DDB based on resolution/width of the display.
4170 total_slice_mask = dbuf_slice_mask;
4171 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4172 const struct drm_display_mode *adjusted_mode =
4173 &crtc_state->hw.adjusted_mode;
4174 enum pipe pipe = crtc->pipe;
4175 int hdisplay, vdisplay;
4176 u32 pipe_dbuf_slice_mask;
4178 if (!crtc_state->hw.active)
4181 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4185 * According to BSpec pipe can share one dbuf slice with another
4186 * pipes or pipe can use multiple dbufs, in both cases we
4187 * account for other pipes only if they have exactly same mask.
4188 * However we need to account how many slices we should enable
4191 total_slice_mask |= pipe_dbuf_slice_mask;
4194 * Do not account pipes using other slice sets
4195 * luckily as of current BSpec slice sets do not partially
4196 * intersect(pipes share either same one slice or same slice set
4197 * i.e no partial intersection), so it is enough to check for
4200 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4203 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4205 total_width_in_range += hdisplay;
4207 if (pipe < for_pipe)
4208 width_before_pipe_in_range += hdisplay;
4209 else if (pipe == for_pipe)
4210 pipe_width = hdisplay;
4214 * FIXME: For now we always enable slice S1 as per
4215 * the Bspec display initialization sequence.
4217 new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1);
4219 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4220 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4225 start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4226 end = ddb_range_size *
4227 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4229 alloc->start = offset + start;
4230 alloc->end = offset + end;
4232 drm_dbg_kms(&dev_priv->drm,
4233 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
4234 for_crtc->base.id, for_crtc->name,
4235 dbuf_slice_mask, alloc->start, alloc->end, active_pipes);
4240 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4241 int width, const struct drm_format_info *format,
4242 u64 modifier, unsigned int rotation,
4243 u32 plane_pixel_rate, struct skl_wm_params *wp,
4245 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4247 unsigned int latency,
4248 const struct skl_wm_params *wp,
4249 const struct skl_wm_level *result_prev,
4250 struct skl_wm_level *result /* out */);
4253 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4256 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4257 int level, max_level = ilk_wm_max_level(dev_priv);
4258 struct skl_wm_level wm = {};
4259 int ret, min_ddb_alloc = 0;
4260 struct skl_wm_params wp;
4262 ret = skl_compute_wm_params(crtc_state, 256,
4263 drm_format_info(DRM_FORMAT_ARGB8888),
4264 DRM_FORMAT_MOD_LINEAR,
4266 crtc_state->pixel_rate, &wp, 0);
4267 drm_WARN_ON(&dev_priv->drm, ret);
4269 for (level = 0; level <= max_level; level++) {
4270 unsigned int latency = dev_priv->wm.skl_latency[level];
4272 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4273 if (wm.min_ddb_alloc == U16_MAX)
4276 min_ddb_alloc = wm.min_ddb_alloc;
4279 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4282 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4283 struct skl_ddb_entry *entry, u32 reg)
4286 entry->start = reg & DDB_ENTRY_MASK;
4287 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4294 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4295 const enum pipe pipe,
4296 const enum plane_id plane_id,
4297 struct skl_ddb_entry *ddb_y,
4298 struct skl_ddb_entry *ddb_uv)
4303 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4304 if (plane_id == PLANE_CURSOR) {
4305 val = I915_READ(CUR_BUF_CFG(pipe));
4306 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4310 val = I915_READ(PLANE_CTL(pipe, plane_id));
4312 /* No DDB allocated for disabled planes */
4313 if (val & PLANE_CTL_ENABLE)
4314 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4315 val & PLANE_CTL_ORDER_RGBX,
4316 val & PLANE_CTL_ALPHA_MASK);
4318 if (INTEL_GEN(dev_priv) >= 11) {
4319 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4320 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4322 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4323 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4326 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4329 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4330 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4334 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4335 struct skl_ddb_entry *ddb_y,
4336 struct skl_ddb_entry *ddb_uv)
4338 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4339 enum intel_display_power_domain power_domain;
4340 enum pipe pipe = crtc->pipe;
4341 intel_wakeref_t wakeref;
4342 enum plane_id plane_id;
4344 power_domain = POWER_DOMAIN_PIPE(pipe);
4345 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4349 for_each_plane_id_on_crtc(crtc, plane_id)
4350 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4355 intel_display_power_put(dev_priv, power_domain, wakeref);
4359 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4360 * The bspec defines downscale amount as:
4363 * Horizontal down scale amount = maximum[1, Horizontal source size /
4364 * Horizontal destination size]
4365 * Vertical down scale amount = maximum[1, Vertical source size /
4366 * Vertical destination size]
4367 * Total down scale amount = Horizontal down scale amount *
4368 * Vertical down scale amount
4371 * Return value is provided in 16.16 fixed point form to retain fractional part.
4372 * Caller should take care of dividing & rounding off the value.
4374 static uint_fixed_16_16_t
4375 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4376 const struct intel_plane_state *plane_state)
4378 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4379 u32 src_w, src_h, dst_w, dst_h;
4380 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4381 uint_fixed_16_16_t downscale_h, downscale_w;
4383 if (drm_WARN_ON(&dev_priv->drm,
4384 !intel_wm_plane_visible(crtc_state, plane_state)))
4385 return u32_to_fixed16(0);
4388 * Src coordinates are already rotated by 270 degrees for
4389 * the 90/270 degree plane rotation cases (to match the
4390 * GTT mapping), hence no need to account for rotation here.
4392 * n.b., src is 16.16 fixed point, dst is whole integer.
4394 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4395 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4396 dst_w = drm_rect_width(&plane_state->uapi.dst);
4397 dst_h = drm_rect_height(&plane_state->uapi.dst);
4399 fp_w_ratio = div_fixed16(src_w, dst_w);
4400 fp_h_ratio = div_fixed16(src_h, dst_h);
4401 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4402 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4404 return mul_fixed16(downscale_w, downscale_h);
4407 struct dbuf_slice_conf_entry {
4409 u8 dbuf_mask[I915_MAX_PIPES];
4413 * Table taken from Bspec 12716
4414 * Pipes do have some preferred DBuf slice affinity,
4415 * plus there are some hardcoded requirements on how
4416 * those should be distributed for multipipe scenarios.
4417 * For more DBuf slices algorithm can get even more messy
4418 * and less readable, so decided to use a table almost
4419 * as is from BSpec itself - that way it is at least easier
4420 * to compare, change and check.
4422 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4423 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4426 .active_pipes = BIT(PIPE_A),
4428 [PIPE_A] = BIT(DBUF_S1),
4432 .active_pipes = BIT(PIPE_B),
4434 [PIPE_B] = BIT(DBUF_S1),
4438 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4440 [PIPE_A] = BIT(DBUF_S1),
4441 [PIPE_B] = BIT(DBUF_S2),
4445 .active_pipes = BIT(PIPE_C),
4447 [PIPE_C] = BIT(DBUF_S2),
4451 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4453 [PIPE_A] = BIT(DBUF_S1),
4454 [PIPE_C] = BIT(DBUF_S2),
4458 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4460 [PIPE_B] = BIT(DBUF_S1),
4461 [PIPE_C] = BIT(DBUF_S2),
4465 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4467 [PIPE_A] = BIT(DBUF_S1),
4468 [PIPE_B] = BIT(DBUF_S1),
4469 [PIPE_C] = BIT(DBUF_S2),
4476 * Table taken from Bspec 49255
4477 * Pipes do have some preferred DBuf slice affinity,
4478 * plus there are some hardcoded requirements on how
4479 * those should be distributed for multipipe scenarios.
4480 * For more DBuf slices algorithm can get even more messy
4481 * and less readable, so decided to use a table almost
4482 * as is from BSpec itself - that way it is at least easier
4483 * to compare, change and check.
4485 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4486 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4489 .active_pipes = BIT(PIPE_A),
4491 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4495 .active_pipes = BIT(PIPE_B),
4497 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4501 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4503 [PIPE_A] = BIT(DBUF_S2),
4504 [PIPE_B] = BIT(DBUF_S1),
4508 .active_pipes = BIT(PIPE_C),
4510 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4514 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4516 [PIPE_A] = BIT(DBUF_S1),
4517 [PIPE_C] = BIT(DBUF_S2),
4521 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4523 [PIPE_B] = BIT(DBUF_S1),
4524 [PIPE_C] = BIT(DBUF_S2),
4528 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4530 [PIPE_A] = BIT(DBUF_S1),
4531 [PIPE_B] = BIT(DBUF_S1),
4532 [PIPE_C] = BIT(DBUF_S2),
4536 .active_pipes = BIT(PIPE_D),
4538 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4542 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4544 [PIPE_A] = BIT(DBUF_S1),
4545 [PIPE_D] = BIT(DBUF_S2),
4549 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4551 [PIPE_B] = BIT(DBUF_S1),
4552 [PIPE_D] = BIT(DBUF_S2),
4556 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4558 [PIPE_A] = BIT(DBUF_S1),
4559 [PIPE_B] = BIT(DBUF_S1),
4560 [PIPE_D] = BIT(DBUF_S2),
4564 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4566 [PIPE_C] = BIT(DBUF_S1),
4567 [PIPE_D] = BIT(DBUF_S2),
4571 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4573 [PIPE_A] = BIT(DBUF_S1),
4574 [PIPE_C] = BIT(DBUF_S2),
4575 [PIPE_D] = BIT(DBUF_S2),
4579 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4581 [PIPE_B] = BIT(DBUF_S1),
4582 [PIPE_C] = BIT(DBUF_S2),
4583 [PIPE_D] = BIT(DBUF_S2),
4587 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4589 [PIPE_A] = BIT(DBUF_S1),
4590 [PIPE_B] = BIT(DBUF_S1),
4591 [PIPE_C] = BIT(DBUF_S2),
4592 [PIPE_D] = BIT(DBUF_S2),
4598 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4599 const struct dbuf_slice_conf_entry *dbuf_slices)
4603 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4604 if (dbuf_slices[i].active_pipes == active_pipes)
4605 return dbuf_slices[i].dbuf_mask[pipe];
4611 * This function finds an entry with same enabled pipe configuration and
4612 * returns correspondent DBuf slice mask as stated in BSpec for particular
4615 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4618 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4619 * required calculating "pipe ratio" in order to determine
4620 * if one or two slices can be used for single pipe configurations
4621 * as additional constraint to the existing table.
4622 * However based on recent info, it should be not "pipe ratio"
4623 * but rather ratio between pixel_rate and cdclk with additional
4624 * constants, so for now we are using only table until this is
4625 * clarified. Also this is the reason why crtc_state param is
4626 * still here - we will need it once those additional constraints
4629 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4632 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4634 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4637 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4640 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4641 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4642 enum pipe pipe = crtc->pipe;
4644 if (IS_GEN(dev_priv, 12))
4645 return tgl_compute_dbuf_slices(pipe, active_pipes);
4646 else if (IS_GEN(dev_priv, 11))
4647 return icl_compute_dbuf_slices(pipe, active_pipes);
4649 * For anything else just return one slice yet.
4650 * Should be extended for other platforms.
4652 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4656 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4657 const struct intel_plane_state *plane_state,
4660 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4661 const struct drm_framebuffer *fb = plane_state->hw.fb;
4663 u32 width = 0, height = 0;
4664 uint_fixed_16_16_t down_scale_amount;
4667 if (!plane_state->uapi.visible)
4670 if (plane->id == PLANE_CURSOR)
4673 if (color_plane == 1 &&
4674 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4678 * Src coordinates are already rotated by 270 degrees for
4679 * the 90/270 degree plane rotation cases (to match the
4680 * GTT mapping), hence no need to account for rotation here.
4682 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4683 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4685 /* UV plane does 1/2 pixel sub-sampling */
4686 if (color_plane == 1) {
4691 data_rate = width * height;
4693 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4695 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4697 rate *= fb->format->cpp[color_plane];
4702 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4703 u64 *plane_data_rate,
4704 u64 *uv_plane_data_rate)
4706 struct intel_plane *plane;
4707 const struct intel_plane_state *plane_state;
4708 u64 total_data_rate = 0;
4710 /* Calculate and cache data rate for each plane */
4711 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4712 enum plane_id plane_id = plane->id;
4716 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4717 plane_data_rate[plane_id] = rate;
4718 total_data_rate += rate;
4721 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4722 uv_plane_data_rate[plane_id] = rate;
4723 total_data_rate += rate;
4726 return total_data_rate;
4730 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4731 u64 *plane_data_rate)
4733 struct intel_plane *plane;
4734 const struct intel_plane_state *plane_state;
4735 u64 total_data_rate = 0;
4737 /* Calculate and cache data rate for each plane */
4738 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4739 enum plane_id plane_id = plane->id;
4742 if (!plane_state->planar_linked_plane) {
4743 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4744 plane_data_rate[plane_id] = rate;
4745 total_data_rate += rate;
4747 enum plane_id y_plane_id;
4750 * The slave plane might not iterate in
4751 * intel_atomic_crtc_state_for_each_plane_state(),
4752 * and needs the master plane state which may be
4753 * NULL if we try get_new_plane_state(), so we
4754 * always calculate from the master.
4756 if (plane_state->planar_slave)
4759 /* Y plane rate is calculated on the slave */
4760 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4761 y_plane_id = plane_state->planar_linked_plane->id;
4762 plane_data_rate[y_plane_id] = rate;
4763 total_data_rate += rate;
4765 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4766 plane_data_rate[plane_id] = rate;
4767 total_data_rate += rate;
4771 return total_data_rate;
4774 static const struct skl_wm_level *
4775 skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4776 enum plane_id plane_id,
4779 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4780 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4782 if (level == 0 && pipe_wm->use_sagv_wm)
4783 return &wm->sagv_wm0;
4785 return &wm->wm[level];
4789 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4791 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4792 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4793 struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4794 u16 alloc_size, start = 0;
4795 u16 total[I915_MAX_PLANES] = {};
4796 u16 uv_total[I915_MAX_PLANES] = {};
4797 u64 total_data_rate;
4798 enum plane_id plane_id;
4800 u64 plane_data_rate[I915_MAX_PLANES] = {};
4801 u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4806 /* Clear the partitioning for disabled planes. */
4807 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4808 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4810 if (!crtc_state->hw.active) {
4811 struct intel_atomic_state *state =
4812 to_intel_atomic_state(crtc_state->uapi.state);
4813 struct intel_dbuf_state *new_dbuf_state =
4814 intel_atomic_get_new_dbuf_state(state);
4815 const struct intel_dbuf_state *old_dbuf_state =
4816 intel_atomic_get_old_dbuf_state(state);
4819 * FIXME hack to make sure we compute this sensibly when
4820 * turning off all the pipes. Otherwise we leave it at
4821 * whatever we had previously, and then runtime PM will
4822 * mess it up by turning off all but S1. Remove this
4823 * once the dbuf state computation flow becomes sane.
4825 if (new_dbuf_state->active_pipes == 0) {
4826 new_dbuf_state->enabled_slices = BIT(DBUF_S1);
4828 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4829 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4835 alloc->start = alloc->end = 0;
4839 if (INTEL_GEN(dev_priv) >= 11)
4841 icl_get_total_relative_data_rate(crtc_state,
4845 skl_get_total_relative_data_rate(crtc_state,
4847 uv_plane_data_rate);
4849 ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state,
4851 alloc, &num_active);
4855 alloc_size = skl_ddb_entry_size(alloc);
4856 if (alloc_size == 0)
4859 /* Allocate fixed number of blocks for cursor. */
4860 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4861 alloc_size -= total[PLANE_CURSOR];
4862 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4863 alloc->end - total[PLANE_CURSOR];
4864 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4866 if (total_data_rate == 0)
4870 * Find the highest watermark level for which we can satisfy the block
4871 * requirement of active planes.
4873 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4875 for_each_plane_id_on_crtc(crtc, plane_id) {
4876 const struct skl_plane_wm *wm =
4877 &crtc_state->wm.skl.optimal.planes[plane_id];
4879 if (plane_id == PLANE_CURSOR) {
4880 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4881 drm_WARN_ON(&dev_priv->drm,
4882 wm->wm[level].min_ddb_alloc != U16_MAX);
4889 blocks += wm->wm[level].min_ddb_alloc;
4890 blocks += wm->uv_wm[level].min_ddb_alloc;
4893 if (blocks <= alloc_size) {
4894 alloc_size -= blocks;
4900 drm_dbg_kms(&dev_priv->drm,
4901 "Requested display configuration exceeds system DDB limitations");
4902 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4903 blocks, alloc_size);
4908 * Grant each plane the blocks it requires at the highest achievable
4909 * watermark level, plus an extra share of the leftover blocks
4910 * proportional to its relative data rate.
4912 for_each_plane_id_on_crtc(crtc, plane_id) {
4913 const struct skl_plane_wm *wm =
4914 &crtc_state->wm.skl.optimal.planes[plane_id];
4918 if (plane_id == PLANE_CURSOR)
4922 * We've accounted for all active planes; remaining planes are
4925 if (total_data_rate == 0)
4928 rate = plane_data_rate[plane_id];
4929 extra = min_t(u16, alloc_size,
4930 DIV64_U64_ROUND_UP(alloc_size * rate,
4932 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4933 alloc_size -= extra;
4934 total_data_rate -= rate;
4936 if (total_data_rate == 0)
4939 rate = uv_plane_data_rate[plane_id];
4940 extra = min_t(u16, alloc_size,
4941 DIV64_U64_ROUND_UP(alloc_size * rate,
4943 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4944 alloc_size -= extra;
4945 total_data_rate -= rate;
4947 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4949 /* Set the actual DDB start/end points for each plane */
4950 start = alloc->start;
4951 for_each_plane_id_on_crtc(crtc, plane_id) {
4952 struct skl_ddb_entry *plane_alloc =
4953 &crtc_state->wm.skl.plane_ddb_y[plane_id];
4954 struct skl_ddb_entry *uv_plane_alloc =
4955 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4957 if (plane_id == PLANE_CURSOR)
4960 /* Gen11+ uses a separate plane for UV watermarks */
4961 drm_WARN_ON(&dev_priv->drm,
4962 INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4964 /* Leave disabled planes at (0,0) */
4965 if (total[plane_id]) {
4966 plane_alloc->start = start;
4967 start += total[plane_id];
4968 plane_alloc->end = start;
4971 if (uv_total[plane_id]) {
4972 uv_plane_alloc->start = start;
4973 start += uv_total[plane_id];
4974 uv_plane_alloc->end = start;
4979 * When we calculated watermark values we didn't know how high
4980 * of a level we'd actually be able to hit, so we just marked
4981 * all levels as "enabled." Go back now and disable the ones
4982 * that aren't actually possible.
4984 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4985 for_each_plane_id_on_crtc(crtc, plane_id) {
4986 struct skl_plane_wm *wm =
4987 &crtc_state->wm.skl.optimal.planes[plane_id];
4990 * We only disable the watermarks for each plane if
4991 * they exceed the ddb allocation of said plane. This
4992 * is done so that we don't end up touching cursor
4993 * watermarks needlessly when some other plane reduces
4994 * our max possible watermark level.
4996 * Bspec has this to say about the PLANE_WM enable bit:
4997 * "All the watermarks at this level for all enabled
4998 * planes must be enabled before the level will be used."
4999 * So this is actually safe to do.
5001 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
5002 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
5003 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
5006 * Wa_1408961008:icl, ehl
5007 * Underruns with WM1+ disabled
5009 if (IS_GEN(dev_priv, 11) &&
5010 level == 1 && wm->wm[0].plane_en) {
5011 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
5012 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
5013 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5019 * Go back and disable the transition watermark if it turns out we
5020 * don't have enough DDB blocks for it.
5022 for_each_plane_id_on_crtc(crtc, plane_id) {
5023 struct skl_plane_wm *wm =
5024 &crtc_state->wm.skl.optimal.planes[plane_id];
5026 if (wm->trans_wm.plane_res_b >= total[plane_id])
5027 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
5034 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5035 * for the read latency) and cpp should always be <= 8, so that
5036 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5037 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5039 static uint_fixed_16_16_t
5040 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5041 u8 cpp, u32 latency, u32 dbuf_block_size)
5043 u32 wm_intermediate_val;
5044 uint_fixed_16_16_t ret;
5047 return FP_16_16_MAX;
5049 wm_intermediate_val = latency * pixel_rate * cpp;
5050 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5052 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5053 ret = add_fixed16_u32(ret, 1);
5058 static uint_fixed_16_16_t
5059 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5060 uint_fixed_16_16_t plane_blocks_per_line)
5062 u32 wm_intermediate_val;
5063 uint_fixed_16_16_t ret;
5066 return FP_16_16_MAX;
5068 wm_intermediate_val = latency * pixel_rate;
5069 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5070 pipe_htotal * 1000);
5071 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5075 static uint_fixed_16_16_t
5076 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5078 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5081 uint_fixed_16_16_t linetime_us;
5083 if (!crtc_state->hw.active)
5084 return u32_to_fixed16(0);
5086 pixel_rate = crtc_state->pixel_rate;
5088 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5089 return u32_to_fixed16(0);
5091 crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5092 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5098 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5099 const struct intel_plane_state *plane_state)
5101 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5102 u64 adjusted_pixel_rate;
5103 uint_fixed_16_16_t downscale_amount;
5105 /* Shouldn't reach here on disabled planes... */
5106 if (drm_WARN_ON(&dev_priv->drm,
5107 !intel_wm_plane_visible(crtc_state, plane_state)))
5111 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5112 * with additional adjustments for plane-specific scaling.
5114 adjusted_pixel_rate = crtc_state->pixel_rate;
5115 downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5117 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5122 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5123 int width, const struct drm_format_info *format,
5124 u64 modifier, unsigned int rotation,
5125 u32 plane_pixel_rate, struct skl_wm_params *wp,
5128 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5129 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5132 /* only planar format has two planes */
5133 if (color_plane == 1 &&
5134 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5135 drm_dbg_kms(&dev_priv->drm,
5136 "Non planar format have single plane\n");
5140 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5141 modifier == I915_FORMAT_MOD_Yf_TILED ||
5142 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5143 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5144 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5145 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5146 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5147 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5150 if (color_plane == 1 && wp->is_planar)
5153 wp->cpp = format->cpp[color_plane];
5154 wp->plane_pixel_rate = plane_pixel_rate;
5156 if (INTEL_GEN(dev_priv) >= 11 &&
5157 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5158 wp->dbuf_block_size = 256;
5160 wp->dbuf_block_size = 512;
5162 if (drm_rotation_90_or_270(rotation)) {
5165 wp->y_min_scanlines = 16;
5168 wp->y_min_scanlines = 8;
5171 wp->y_min_scanlines = 4;
5174 MISSING_CASE(wp->cpp);
5178 wp->y_min_scanlines = 4;
5181 if (skl_needs_memory_bw_wa(dev_priv))
5182 wp->y_min_scanlines *= 2;
5184 wp->plane_bytes_per_line = wp->width * wp->cpp;
5186 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5187 wp->y_min_scanlines,
5188 wp->dbuf_block_size);
5190 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5193 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5194 wp->y_min_scanlines);
5196 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5197 wp->dbuf_block_size);
5200 INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5203 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5206 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5207 wp->plane_blocks_per_line);
5209 wp->linetime_us = fixed16_to_u32_round_up(
5210 intel_get_linetime_us(crtc_state));
5216 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5217 const struct intel_plane_state *plane_state,
5218 struct skl_wm_params *wp, int color_plane)
5220 const struct drm_framebuffer *fb = plane_state->hw.fb;
5224 * Src coordinates are already rotated by 270 degrees for
5225 * the 90/270 degree plane rotation cases (to match the
5226 * GTT mapping), hence no need to account for rotation here.
5228 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5230 return skl_compute_wm_params(crtc_state, width,
5231 fb->format, fb->modifier,
5232 plane_state->hw.rotation,
5233 skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5237 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5239 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5242 /* The number of lines are ignored for the level 0 watermark. */
5246 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5248 unsigned int latency,
5249 const struct skl_wm_params *wp,
5250 const struct skl_wm_level *result_prev,
5251 struct skl_wm_level *result /* out */)
5253 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5254 uint_fixed_16_16_t method1, method2;
5255 uint_fixed_16_16_t selected_result;
5256 u32 res_blocks, res_lines, min_ddb_alloc = 0;
5260 result->min_ddb_alloc = U16_MAX;
5265 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5266 * Display WA #1141: kbl,cfl
5268 if ((IS_KABYLAKE(dev_priv) ||
5269 IS_COFFEELAKE(dev_priv) ||
5270 IS_COMETLAKE(dev_priv)) &&
5271 dev_priv->ipc_enabled)
5274 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5277 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5278 wp->cpp, latency, wp->dbuf_block_size);
5279 method2 = skl_wm_method2(wp->plane_pixel_rate,
5280 crtc_state->hw.adjusted_mode.crtc_htotal,
5282 wp->plane_blocks_per_line);
5285 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5287 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5288 wp->dbuf_block_size < 1) &&
5289 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5290 selected_result = method2;
5291 } else if (latency >= wp->linetime_us) {
5292 if (IS_GEN(dev_priv, 9) &&
5293 !IS_GEMINILAKE(dev_priv))
5294 selected_result = min_fixed16(method1, method2);
5296 selected_result = method2;
5298 selected_result = method1;
5302 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5303 res_lines = div_round_up_fixed16(selected_result,
5304 wp->plane_blocks_per_line);
5306 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5307 /* Display WA #1125: skl,bxt,kbl */
5308 if (level == 0 && wp->rc_surface)
5310 fixed16_to_u32_round_up(wp->y_tile_minimum);
5312 /* Display WA #1126: skl,bxt,kbl */
5313 if (level >= 1 && level <= 7) {
5316 fixed16_to_u32_round_up(wp->y_tile_minimum);
5317 res_lines += wp->y_min_scanlines;
5323 * Make sure result blocks for higher latency levels are
5324 * atleast as high as level below the current level.
5325 * Assumption in DDB algorithm optimization for special
5326 * cases. Also covers Display WA #1125 for RC.
5328 if (result_prev->plane_res_b > res_blocks)
5329 res_blocks = result_prev->plane_res_b;
5333 if (INTEL_GEN(dev_priv) >= 11) {
5337 if (res_lines % wp->y_min_scanlines == 0)
5338 extra_lines = wp->y_min_scanlines;
5340 extra_lines = wp->y_min_scanlines * 2 -
5341 res_lines % wp->y_min_scanlines;
5343 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5344 wp->plane_blocks_per_line);
5346 min_ddb_alloc = res_blocks +
5347 DIV_ROUND_UP(res_blocks, 10);
5351 if (!skl_wm_has_lines(dev_priv, level))
5354 if (res_lines > 31) {
5356 result->min_ddb_alloc = U16_MAX;
5361 * If res_lines is valid, assume we can use this watermark level
5362 * for now. We'll come back and disable it after we calculate the
5363 * DDB allocation if it turns out we don't actually have enough
5364 * blocks to satisfy it.
5366 result->plane_res_b = res_blocks;
5367 result->plane_res_l = res_lines;
5368 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5369 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5370 result->plane_en = true;
5374 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5375 const struct skl_wm_params *wm_params,
5376 struct skl_wm_level *levels)
5378 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5379 int level, max_level = ilk_wm_max_level(dev_priv);
5380 struct skl_wm_level *result_prev = &levels[0];
5382 for (level = 0; level <= max_level; level++) {
5383 struct skl_wm_level *result = &levels[level];
5384 unsigned int latency = dev_priv->wm.skl_latency[level];
5386 skl_compute_plane_wm(crtc_state, level, latency,
5387 wm_params, result_prev, result);
5389 result_prev = result;
5393 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5394 const struct skl_wm_params *wm_params,
5395 struct skl_plane_wm *plane_wm)
5397 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5398 struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5399 struct skl_wm_level *levels = plane_wm->wm;
5400 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5402 skl_compute_plane_wm(crtc_state, 0, latency,
5403 wm_params, &levels[0],
5407 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5408 const struct skl_wm_params *wp,
5409 struct skl_plane_wm *wm)
5411 struct drm_device *dev = crtc_state->uapi.crtc->dev;
5412 const struct drm_i915_private *dev_priv = to_i915(dev);
5413 u16 trans_min, trans_amount, trans_y_tile_min;
5414 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5416 /* Transition WM don't make any sense if ipc is disabled */
5417 if (!dev_priv->ipc_enabled)
5421 * WaDisableTWM:skl,kbl,cfl,bxt
5422 * Transition WM are not recommended by HW team for GEN9
5424 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5427 if (INTEL_GEN(dev_priv) >= 11)
5432 /* Display WA #1140: glk,cnl */
5433 if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5436 trans_amount = 10; /* This is configurable amount */
5438 trans_offset_b = trans_min + trans_amount;
5441 * The spec asks for Selected Result Blocks for wm0 (the real value),
5442 * not Result Blocks (the integer value). Pay attention to the capital
5443 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5444 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5445 * and since we later will have to get the ceiling of the sum in the
5446 * transition watermarks calculation, we can just pretend Selected
5447 * Result Blocks is Result Blocks minus 1 and it should work for the
5448 * current platforms.
5450 wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5454 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5455 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5458 res_blocks = wm0_sel_res_b + trans_offset_b;
5462 * Just assume we can enable the transition watermark. After
5463 * computing the DDB we'll come back and disable it if that
5464 * assumption turns out to be false.
5466 wm->trans_wm.plane_res_b = res_blocks + 1;
5467 wm->trans_wm.plane_en = true;
5470 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5471 const struct intel_plane_state *plane_state,
5472 enum plane_id plane_id, int color_plane)
5474 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5475 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5476 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5477 struct skl_wm_params wm_params;
5480 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5481 &wm_params, color_plane);
5485 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5487 if (INTEL_GEN(dev_priv) >= 12)
5488 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5490 skl_compute_transition_wm(crtc_state, &wm_params, wm);
5495 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5496 const struct intel_plane_state *plane_state,
5497 enum plane_id plane_id)
5499 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5500 struct skl_wm_params wm_params;
5503 wm->is_planar = true;
5505 /* uv plane watermarks must also be validated for NV12/Planar */
5506 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5511 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5516 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5517 const struct intel_plane_state *plane_state)
5519 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5520 const struct drm_framebuffer *fb = plane_state->hw.fb;
5521 enum plane_id plane_id = plane->id;
5524 if (!intel_wm_plane_visible(crtc_state, plane_state))
5527 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5532 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5533 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5542 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5543 const struct intel_plane_state *plane_state)
5545 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5546 enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5549 /* Watermarks calculated in master */
5550 if (plane_state->planar_slave)
5553 if (plane_state->planar_linked_plane) {
5554 const struct drm_framebuffer *fb = plane_state->hw.fb;
5555 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5557 drm_WARN_ON(&dev_priv->drm,
5558 !intel_wm_plane_visible(crtc_state, plane_state));
5559 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5560 fb->format->num_planes == 1);
5562 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5567 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5571 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5572 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5581 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5583 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5584 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5585 struct intel_plane *plane;
5586 const struct intel_plane_state *plane_state;
5590 * We'll only calculate watermarks for planes that are actually
5591 * enabled, so make sure all other planes are set as disabled.
5593 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5595 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5598 if (INTEL_GEN(dev_priv) >= 11)
5599 ret = icl_build_plane_wm(crtc_state, plane_state);
5601 ret = skl_build_plane_wm(crtc_state, plane_state);
5609 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5611 const struct skl_ddb_entry *entry)
5614 intel_de_write_fw(dev_priv, reg,
5615 (entry->end - 1) << 16 | entry->start);
5617 intel_de_write_fw(dev_priv, reg, 0);
5620 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5622 const struct skl_wm_level *level)
5626 if (level->plane_en)
5628 if (level->ignore_lines)
5629 val |= PLANE_WM_IGNORE_LINES;
5630 val |= level->plane_res_b;
5631 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5633 intel_de_write_fw(dev_priv, reg, val);
5636 void skl_write_plane_wm(struct intel_plane *plane,
5637 const struct intel_crtc_state *crtc_state)
5639 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5640 int level, max_level = ilk_wm_max_level(dev_priv);
5641 enum plane_id plane_id = plane->id;
5642 enum pipe pipe = plane->pipe;
5643 const struct skl_plane_wm *wm =
5644 &crtc_state->wm.skl.optimal.planes[plane_id];
5645 const struct skl_ddb_entry *ddb_y =
5646 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5647 const struct skl_ddb_entry *ddb_uv =
5648 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5650 for (level = 0; level <= max_level; level++) {
5651 const struct skl_wm_level *wm_level;
5653 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5655 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5658 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5661 if (INTEL_GEN(dev_priv) >= 11) {
5662 skl_ddb_entry_write(dev_priv,
5663 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5668 swap(ddb_y, ddb_uv);
5670 skl_ddb_entry_write(dev_priv,
5671 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5672 skl_ddb_entry_write(dev_priv,
5673 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5676 void skl_write_cursor_wm(struct intel_plane *plane,
5677 const struct intel_crtc_state *crtc_state)
5679 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5680 int level, max_level = ilk_wm_max_level(dev_priv);
5681 enum plane_id plane_id = plane->id;
5682 enum pipe pipe = plane->pipe;
5683 const struct skl_plane_wm *wm =
5684 &crtc_state->wm.skl.optimal.planes[plane_id];
5685 const struct skl_ddb_entry *ddb =
5686 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5688 for (level = 0; level <= max_level; level++) {
5689 const struct skl_wm_level *wm_level;
5691 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5693 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5696 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5698 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5701 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5702 const struct skl_wm_level *l2)
5704 return l1->plane_en == l2->plane_en &&
5705 l1->ignore_lines == l2->ignore_lines &&
5706 l1->plane_res_l == l2->plane_res_l &&
5707 l1->plane_res_b == l2->plane_res_b;
5710 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5711 const struct skl_plane_wm *wm1,
5712 const struct skl_plane_wm *wm2)
5714 int level, max_level = ilk_wm_max_level(dev_priv);
5716 for (level = 0; level <= max_level; level++) {
5718 * We don't check uv_wm as the hardware doesn't actually
5719 * use it. It only gets used for calculating the required
5722 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5726 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5729 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5730 const struct skl_ddb_entry *b)
5732 return a->start < b->end && b->start < a->end;
5735 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5736 const struct skl_ddb_entry *entries,
5737 int num_entries, int ignore_idx)
5741 for (i = 0; i < num_entries; i++) {
5742 if (i != ignore_idx &&
5743 skl_ddb_entries_overlap(ddb, &entries[i]))
5751 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5752 struct intel_crtc_state *new_crtc_state)
5754 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5755 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5756 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5757 struct intel_plane *plane;
5759 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5760 struct intel_plane_state *plane_state;
5761 enum plane_id plane_id = plane->id;
5763 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5764 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5765 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5766 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5769 plane_state = intel_atomic_get_plane_state(state, plane);
5770 if (IS_ERR(plane_state))
5771 return PTR_ERR(plane_state);
5773 new_crtc_state->update_planes |= BIT(plane_id);
5780 skl_compute_ddb(struct intel_atomic_state *state)
5782 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5783 const struct intel_dbuf_state *old_dbuf_state;
5784 const struct intel_dbuf_state *new_dbuf_state;
5785 const struct intel_crtc_state *old_crtc_state;
5786 struct intel_crtc_state *new_crtc_state;
5787 struct intel_crtc *crtc;
5790 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5791 new_crtc_state, i) {
5792 ret = skl_allocate_pipe_ddb(new_crtc_state);
5796 ret = skl_ddb_add_affected_planes(old_crtc_state,
5802 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5803 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
5805 if (new_dbuf_state &&
5806 new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices)
5807 drm_dbg_kms(&dev_priv->drm,
5808 "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5809 old_dbuf_state->enabled_slices,
5810 new_dbuf_state->enabled_slices,
5811 INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5816 static char enast(bool enable)
5818 return enable ? '*' : ' ';
5822 skl_print_wm_changes(struct intel_atomic_state *state)
5824 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5825 const struct intel_crtc_state *old_crtc_state;
5826 const struct intel_crtc_state *new_crtc_state;
5827 struct intel_plane *plane;
5828 struct intel_crtc *crtc;
5831 if (!drm_debug_enabled(DRM_UT_KMS))
5834 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5835 new_crtc_state, i) {
5836 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5838 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5839 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5841 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5842 enum plane_id plane_id = plane->id;
5843 const struct skl_ddb_entry *old, *new;
5845 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5846 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5848 if (skl_ddb_entry_equal(old, new))
5851 drm_dbg_kms(&dev_priv->drm,
5852 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5853 plane->base.base.id, plane->base.name,
5854 old->start, old->end, new->start, new->end,
5855 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5858 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5859 enum plane_id plane_id = plane->id;
5860 const struct skl_plane_wm *old_wm, *new_wm;
5862 old_wm = &old_pipe_wm->planes[plane_id];
5863 new_wm = &new_pipe_wm->planes[plane_id];
5865 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5868 drm_dbg_kms(&dev_priv->drm,
5869 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5870 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5871 plane->base.base.id, plane->base.name,
5872 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5873 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5874 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5875 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5876 enast(old_wm->trans_wm.plane_en),
5877 enast(old_wm->sagv_wm0.plane_en),
5878 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5879 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5880 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5881 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5882 enast(new_wm->trans_wm.plane_en),
5883 enast(new_wm->sagv_wm0.plane_en));
5885 drm_dbg_kms(&dev_priv->drm,
5886 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5887 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5888 plane->base.base.id, plane->base.name,
5889 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5890 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5891 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5892 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5893 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5894 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5895 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5896 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5897 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5898 enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5900 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5901 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5902 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5903 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5904 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5905 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5906 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5907 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5908 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5909 enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5911 drm_dbg_kms(&dev_priv->drm,
5912 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5913 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5914 plane->base.base.id, plane->base.name,
5915 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5916 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5917 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5918 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5919 old_wm->trans_wm.plane_res_b,
5920 old_wm->sagv_wm0.plane_res_b,
5921 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5922 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5923 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5924 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5925 new_wm->trans_wm.plane_res_b,
5926 new_wm->sagv_wm0.plane_res_b);
5928 drm_dbg_kms(&dev_priv->drm,
5929 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5930 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5931 plane->base.base.id, plane->base.name,
5932 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5933 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5934 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5935 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5936 old_wm->trans_wm.min_ddb_alloc,
5937 old_wm->sagv_wm0.min_ddb_alloc,
5938 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5939 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5940 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5941 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5942 new_wm->trans_wm.min_ddb_alloc,
5943 new_wm->sagv_wm0.min_ddb_alloc);
5948 static int intel_add_affected_pipes(struct intel_atomic_state *state,
5951 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5952 struct intel_crtc *crtc;
5954 for_each_intel_crtc(&dev_priv->drm, crtc) {
5955 struct intel_crtc_state *crtc_state;
5957 if ((pipe_mask & BIT(crtc->pipe)) == 0)
5960 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5961 if (IS_ERR(crtc_state))
5962 return PTR_ERR(crtc_state);
5969 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5971 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5972 struct intel_crtc_state *crtc_state;
5973 struct intel_crtc *crtc;
5976 if (dev_priv->wm.distrust_bios_wm) {
5978 * skl_ddb_get_pipe_allocation_limits() currently requires
5979 * all active pipes to be included in the state so that
5980 * it can redistribute the dbuf among them, and it really
5981 * wants to recompute things when distrust_bios_wm is set
5982 * so we add all the pipes to the state.
5984 ret = intel_add_affected_pipes(state, ~0);
5989 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
5990 struct intel_dbuf_state *new_dbuf_state;
5991 const struct intel_dbuf_state *old_dbuf_state;
5993 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5994 if (IS_ERR(new_dbuf_state))
5995 return PTR_ERR(new_dbuf_state);
5997 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5999 new_dbuf_state->active_pipes =
6000 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6002 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes)
6005 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6010 * skl_ddb_get_pipe_allocation_limits() currently requires
6011 * all active pipes to be included in the state so that
6012 * it can redistribute the dbuf among them.
6014 ret = intel_add_affected_pipes(state,
6015 new_dbuf_state->active_pipes);
6026 * To make sure the cursor watermark registers are always consistent
6027 * with our computed state the following scenario needs special
6031 * 2. move cursor entirely offscreen
6034 * Step 2. does call .disable_plane() but does not zero the watermarks
6035 * (since we consider an offscreen cursor still active for the purposes
6036 * of watermarks). Step 3. would not normally call .disable_plane()
6037 * because the actual plane visibility isn't changing, and we don't
6038 * deallocate the cursor ddb until the pipe gets disabled. So we must
6039 * force step 3. to call .disable_plane() to update the watermark
6040 * registers properly.
6042 * Other planes do not suffer from this issues as their watermarks are
6043 * calculated based on the actual plane visibility. The only time this
6044 * can trigger for the other planes is during the initial readout as the
6045 * default value of the watermarks registers is not zero.
6047 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6048 struct intel_crtc *crtc)
6050 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6051 const struct intel_crtc_state *old_crtc_state =
6052 intel_atomic_get_old_crtc_state(state, crtc);
6053 struct intel_crtc_state *new_crtc_state =
6054 intel_atomic_get_new_crtc_state(state, crtc);
6055 struct intel_plane *plane;
6057 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6058 struct intel_plane_state *plane_state;
6059 enum plane_id plane_id = plane->id;
6062 * Force a full wm update for every plane on modeset.
6063 * Required because the reset value of the wm registers
6064 * is non-zero, whereas we want all disabled planes to
6065 * have zero watermarks. So if we turn off the relevant
6066 * power well the hardware state will go out of sync
6067 * with the software state.
6069 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6070 skl_plane_wm_equals(dev_priv,
6071 &old_crtc_state->wm.skl.optimal.planes[plane_id],
6072 &new_crtc_state->wm.skl.optimal.planes[plane_id]))
6075 plane_state = intel_atomic_get_plane_state(state, plane);
6076 if (IS_ERR(plane_state))
6077 return PTR_ERR(plane_state);
6079 new_crtc_state->update_planes |= BIT(plane_id);
6086 skl_compute_wm(struct intel_atomic_state *state)
6088 struct intel_crtc *crtc;
6089 struct intel_crtc_state *new_crtc_state;
6090 struct intel_crtc_state *old_crtc_state;
6093 ret = skl_ddb_add_affected_pipes(state);
6098 * Calculate WM's for all pipes that are part of this transaction.
6099 * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6100 * weren't otherwise being modified if pipe allocations had to change.
6102 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6103 new_crtc_state, i) {
6104 ret = skl_build_pipe_wm(new_crtc_state);
6109 ret = skl_compute_ddb(state);
6113 ret = intel_compute_sagv_mask(state);
6118 * skl_compute_ddb() will have adjusted the final watermarks
6119 * based on how much ddb is available. Now we can actually
6120 * check if the final watermarks changed.
6122 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6123 new_crtc_state, i) {
6124 ret = skl_wm_add_affected_planes(state, crtc);
6129 skl_print_wm_changes(state);
6134 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6135 struct intel_wm_config *config)
6137 struct intel_crtc *crtc;
6139 /* Compute the currently _active_ config */
6140 for_each_intel_crtc(&dev_priv->drm, crtc) {
6141 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6143 if (!wm->pipe_enabled)
6146 config->sprites_enabled |= wm->sprites_enabled;
6147 config->sprites_scaled |= wm->sprites_scaled;
6148 config->num_pipes_active++;
6152 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6154 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6155 struct ilk_wm_maximums max;
6156 struct intel_wm_config config = {};
6157 struct ilk_wm_values results = {};
6158 enum intel_ddb_partitioning partitioning;
6160 ilk_compute_wm_config(dev_priv, &config);
6162 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6163 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6165 /* 5/6 split only in single pipe config on IVB+ */
6166 if (INTEL_GEN(dev_priv) >= 7 &&
6167 config.num_pipes_active == 1 && config.sprites_enabled) {
6168 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6169 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6171 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6173 best_lp_wm = &lp_wm_1_2;
6176 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6177 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6179 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6181 ilk_write_wm_values(dev_priv, &results);
6184 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6185 struct intel_crtc *crtc)
6187 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6188 const struct intel_crtc_state *crtc_state =
6189 intel_atomic_get_new_crtc_state(state, crtc);
6191 mutex_lock(&dev_priv->wm.wm_mutex);
6192 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6193 ilk_program_watermarks(dev_priv);
6194 mutex_unlock(&dev_priv->wm.wm_mutex);
6197 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6198 struct intel_crtc *crtc)
6200 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6201 const struct intel_crtc_state *crtc_state =
6202 intel_atomic_get_new_crtc_state(state, crtc);
6204 if (!crtc_state->wm.need_postvbl_update)
6207 mutex_lock(&dev_priv->wm.wm_mutex);
6208 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6209 ilk_program_watermarks(dev_priv);
6210 mutex_unlock(&dev_priv->wm.wm_mutex);
6213 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6215 level->plane_en = val & PLANE_WM_EN;
6216 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6217 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6218 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6219 PLANE_WM_LINES_MASK;
6222 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6223 struct skl_pipe_wm *out)
6225 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6226 enum pipe pipe = crtc->pipe;
6227 int level, max_level;
6228 enum plane_id plane_id;
6231 max_level = ilk_wm_max_level(dev_priv);
6233 for_each_plane_id_on_crtc(crtc, plane_id) {
6234 struct skl_plane_wm *wm = &out->planes[plane_id];
6236 for (level = 0; level <= max_level; level++) {
6237 if (plane_id != PLANE_CURSOR)
6238 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6240 val = I915_READ(CUR_WM(pipe, level));
6242 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6245 if (INTEL_GEN(dev_priv) >= 12)
6246 wm->sagv_wm0 = wm->wm[0];
6248 if (plane_id != PLANE_CURSOR)
6249 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6251 val = I915_READ(CUR_WM_TRANS(pipe));
6253 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6260 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6262 struct intel_crtc *crtc;
6263 struct intel_crtc_state *crtc_state;
6265 for_each_intel_crtc(&dev_priv->drm, crtc) {
6266 crtc_state = to_intel_crtc_state(crtc->base.state);
6268 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6271 if (dev_priv->active_pipes) {
6272 /* Fully recompute DDB on first atomic commit */
6273 dev_priv->wm.distrust_bios_wm = true;
6277 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6279 struct drm_device *dev = crtc->base.dev;
6280 struct drm_i915_private *dev_priv = to_i915(dev);
6281 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6282 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6283 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6284 enum pipe pipe = crtc->pipe;
6285 static const i915_reg_t wm0_pipe_reg[] = {
6286 [PIPE_A] = WM0_PIPEA_ILK,
6287 [PIPE_B] = WM0_PIPEB_ILK,
6288 [PIPE_C] = WM0_PIPEC_IVB,
6291 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6293 memset(active, 0, sizeof(*active));
6295 active->pipe_enabled = crtc->active;
6297 if (active->pipe_enabled) {
6298 u32 tmp = hw->wm_pipe[pipe];
6301 * For active pipes LP0 watermark is marked as
6302 * enabled, and LP1+ watermaks as disabled since
6303 * we can't really reverse compute them in case
6304 * multiple pipes are active.
6306 active->wm[0].enable = true;
6307 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6308 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6309 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6311 int level, max_level = ilk_wm_max_level(dev_priv);
6314 * For inactive pipes, all watermark levels
6315 * should be marked as enabled but zeroed,
6316 * which is what we'd compute them to.
6318 for (level = 0; level <= max_level; level++)
6319 active->wm[level].enable = true;
6322 crtc->wm.active.ilk = *active;
6325 #define _FW_WM(value, plane) \
6326 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6327 #define _FW_WM_VLV(value, plane) \
6328 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6330 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6331 struct g4x_wm_values *wm)
6335 tmp = I915_READ(DSPFW1);
6336 wm->sr.plane = _FW_WM(tmp, SR);
6337 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6338 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6339 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6341 tmp = I915_READ(DSPFW2);
6342 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6343 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6344 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6345 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6346 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6347 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6349 tmp = I915_READ(DSPFW3);
6350 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6351 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6352 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6353 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6356 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6357 struct vlv_wm_values *wm)
6362 for_each_pipe(dev_priv, pipe) {
6363 tmp = I915_READ(VLV_DDL(pipe));
6365 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6366 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6367 wm->ddl[pipe].plane[PLANE_CURSOR] =
6368 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6369 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6370 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6371 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6372 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6375 tmp = I915_READ(DSPFW1);
6376 wm->sr.plane = _FW_WM(tmp, SR);
6377 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6378 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6379 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6381 tmp = I915_READ(DSPFW2);
6382 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6383 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6384 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6386 tmp = I915_READ(DSPFW3);
6387 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6389 if (IS_CHERRYVIEW(dev_priv)) {
6390 tmp = I915_READ(DSPFW7_CHV);
6391 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6392 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6394 tmp = I915_READ(DSPFW8_CHV);
6395 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6396 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6398 tmp = I915_READ(DSPFW9_CHV);
6399 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6400 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6402 tmp = I915_READ(DSPHOWM);
6403 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6404 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6405 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6406 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6407 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6408 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6409 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6410 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6411 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6412 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6414 tmp = I915_READ(DSPFW7);
6415 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6416 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6418 tmp = I915_READ(DSPHOWM);
6419 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6420 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6421 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6422 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6423 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6424 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6425 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6432 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6434 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6435 struct intel_crtc *crtc;
6437 g4x_read_wm_values(dev_priv, wm);
6439 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6441 for_each_intel_crtc(&dev_priv->drm, crtc) {
6442 struct intel_crtc_state *crtc_state =
6443 to_intel_crtc_state(crtc->base.state);
6444 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6445 struct g4x_pipe_wm *raw;
6446 enum pipe pipe = crtc->pipe;
6447 enum plane_id plane_id;
6448 int level, max_level;
6450 active->cxsr = wm->cxsr;
6451 active->hpll_en = wm->hpll_en;
6452 active->fbc_en = wm->fbc_en;
6454 active->sr = wm->sr;
6455 active->hpll = wm->hpll;
6457 for_each_plane_id_on_crtc(crtc, plane_id) {
6458 active->wm.plane[plane_id] =
6459 wm->pipe[pipe].plane[plane_id];
6462 if (wm->cxsr && wm->hpll_en)
6463 max_level = G4X_WM_LEVEL_HPLL;
6465 max_level = G4X_WM_LEVEL_SR;
6467 max_level = G4X_WM_LEVEL_NORMAL;
6469 level = G4X_WM_LEVEL_NORMAL;
6470 raw = &crtc_state->wm.g4x.raw[level];
6471 for_each_plane_id_on_crtc(crtc, plane_id)
6472 raw->plane[plane_id] = active->wm.plane[plane_id];
6474 if (++level > max_level)
6477 raw = &crtc_state->wm.g4x.raw[level];
6478 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6479 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6480 raw->plane[PLANE_SPRITE0] = 0;
6481 raw->fbc = active->sr.fbc;
6483 if (++level > max_level)
6486 raw = &crtc_state->wm.g4x.raw[level];
6487 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6488 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6489 raw->plane[PLANE_SPRITE0] = 0;
6490 raw->fbc = active->hpll.fbc;
6493 for_each_plane_id_on_crtc(crtc, plane_id)
6494 g4x_raw_plane_wm_set(crtc_state, level,
6495 plane_id, USHRT_MAX);
6496 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6498 crtc_state->wm.g4x.optimal = *active;
6499 crtc_state->wm.g4x.intermediate = *active;
6501 drm_dbg_kms(&dev_priv->drm,
6502 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6504 wm->pipe[pipe].plane[PLANE_PRIMARY],
6505 wm->pipe[pipe].plane[PLANE_CURSOR],
6506 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6509 drm_dbg_kms(&dev_priv->drm,
6510 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6511 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6512 drm_dbg_kms(&dev_priv->drm,
6513 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6514 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6515 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6516 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6519 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6521 struct intel_plane *plane;
6522 struct intel_crtc *crtc;
6524 mutex_lock(&dev_priv->wm.wm_mutex);
6526 for_each_intel_plane(&dev_priv->drm, plane) {
6527 struct intel_crtc *crtc =
6528 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6529 struct intel_crtc_state *crtc_state =
6530 to_intel_crtc_state(crtc->base.state);
6531 struct intel_plane_state *plane_state =
6532 to_intel_plane_state(plane->base.state);
6533 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6534 enum plane_id plane_id = plane->id;
6537 if (plane_state->uapi.visible)
6540 for (level = 0; level < 3; level++) {
6541 struct g4x_pipe_wm *raw =
6542 &crtc_state->wm.g4x.raw[level];
6544 raw->plane[plane_id] = 0;
6545 wm_state->wm.plane[plane_id] = 0;
6548 if (plane_id == PLANE_PRIMARY) {
6549 for (level = 0; level < 3; level++) {
6550 struct g4x_pipe_wm *raw =
6551 &crtc_state->wm.g4x.raw[level];
6555 wm_state->sr.fbc = 0;
6556 wm_state->hpll.fbc = 0;
6557 wm_state->fbc_en = false;
6561 for_each_intel_crtc(&dev_priv->drm, crtc) {
6562 struct intel_crtc_state *crtc_state =
6563 to_intel_crtc_state(crtc->base.state);
6565 crtc_state->wm.g4x.intermediate =
6566 crtc_state->wm.g4x.optimal;
6567 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6570 g4x_program_watermarks(dev_priv);
6572 mutex_unlock(&dev_priv->wm.wm_mutex);
6575 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6577 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6578 struct intel_crtc *crtc;
6581 vlv_read_wm_values(dev_priv, wm);
6583 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6584 wm->level = VLV_WM_LEVEL_PM2;
6586 if (IS_CHERRYVIEW(dev_priv)) {
6587 vlv_punit_get(dev_priv);
6589 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6590 if (val & DSP_MAXFIFO_PM5_ENABLE)
6591 wm->level = VLV_WM_LEVEL_PM5;
6594 * If DDR DVFS is disabled in the BIOS, Punit
6595 * will never ack the request. So if that happens
6596 * assume we don't have to enable/disable DDR DVFS
6597 * dynamically. To test that just set the REQ_ACK
6598 * bit to poke the Punit, but don't change the
6599 * HIGH/LOW bits so that we don't actually change
6600 * the current state.
6602 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6603 val |= FORCE_DDR_FREQ_REQ_ACK;
6604 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6606 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6607 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6608 drm_dbg_kms(&dev_priv->drm,
6609 "Punit not acking DDR DVFS request, "
6610 "assuming DDR DVFS is disabled\n");
6611 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6613 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6614 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6615 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6618 vlv_punit_put(dev_priv);
6621 for_each_intel_crtc(&dev_priv->drm, crtc) {
6622 struct intel_crtc_state *crtc_state =
6623 to_intel_crtc_state(crtc->base.state);
6624 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6625 const struct vlv_fifo_state *fifo_state =
6626 &crtc_state->wm.vlv.fifo_state;
6627 enum pipe pipe = crtc->pipe;
6628 enum plane_id plane_id;
6631 vlv_get_fifo_size(crtc_state);
6633 active->num_levels = wm->level + 1;
6634 active->cxsr = wm->cxsr;
6636 for (level = 0; level < active->num_levels; level++) {
6637 struct g4x_pipe_wm *raw =
6638 &crtc_state->wm.vlv.raw[level];
6640 active->sr[level].plane = wm->sr.plane;
6641 active->sr[level].cursor = wm->sr.cursor;
6643 for_each_plane_id_on_crtc(crtc, plane_id) {
6644 active->wm[level].plane[plane_id] =
6645 wm->pipe[pipe].plane[plane_id];
6647 raw->plane[plane_id] =
6648 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6649 fifo_state->plane[plane_id]);
6653 for_each_plane_id_on_crtc(crtc, plane_id)
6654 vlv_raw_plane_wm_set(crtc_state, level,
6655 plane_id, USHRT_MAX);
6656 vlv_invalidate_wms(crtc, active, level);
6658 crtc_state->wm.vlv.optimal = *active;
6659 crtc_state->wm.vlv.intermediate = *active;
6661 drm_dbg_kms(&dev_priv->drm,
6662 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6664 wm->pipe[pipe].plane[PLANE_PRIMARY],
6665 wm->pipe[pipe].plane[PLANE_CURSOR],
6666 wm->pipe[pipe].plane[PLANE_SPRITE0],
6667 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6670 drm_dbg_kms(&dev_priv->drm,
6671 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6672 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6675 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6677 struct intel_plane *plane;
6678 struct intel_crtc *crtc;
6680 mutex_lock(&dev_priv->wm.wm_mutex);
6682 for_each_intel_plane(&dev_priv->drm, plane) {
6683 struct intel_crtc *crtc =
6684 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6685 struct intel_crtc_state *crtc_state =
6686 to_intel_crtc_state(crtc->base.state);
6687 struct intel_plane_state *plane_state =
6688 to_intel_plane_state(plane->base.state);
6689 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6690 const struct vlv_fifo_state *fifo_state =
6691 &crtc_state->wm.vlv.fifo_state;
6692 enum plane_id plane_id = plane->id;
6695 if (plane_state->uapi.visible)
6698 for (level = 0; level < wm_state->num_levels; level++) {
6699 struct g4x_pipe_wm *raw =
6700 &crtc_state->wm.vlv.raw[level];
6702 raw->plane[plane_id] = 0;
6704 wm_state->wm[level].plane[plane_id] =
6705 vlv_invert_wm_value(raw->plane[plane_id],
6706 fifo_state->plane[plane_id]);
6710 for_each_intel_crtc(&dev_priv->drm, crtc) {
6711 struct intel_crtc_state *crtc_state =
6712 to_intel_crtc_state(crtc->base.state);
6714 crtc_state->wm.vlv.intermediate =
6715 crtc_state->wm.vlv.optimal;
6716 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6719 vlv_program_watermarks(dev_priv);
6721 mutex_unlock(&dev_priv->wm.wm_mutex);
6725 * FIXME should probably kill this and improve
6726 * the real watermark readout/sanitation instead
6728 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6730 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6731 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6732 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6735 * Don't touch WM1S_LP_EN here.
6736 * Doing so could cause underruns.
6740 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6742 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6743 struct intel_crtc *crtc;
6745 ilk_init_lp_watermarks(dev_priv);
6747 for_each_intel_crtc(&dev_priv->drm, crtc)
6748 ilk_pipe_wm_get_hw_state(crtc);
6750 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6751 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6752 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6754 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6755 if (INTEL_GEN(dev_priv) >= 7) {
6756 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6757 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6760 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6761 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6762 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6763 else if (IS_IVYBRIDGE(dev_priv))
6764 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6765 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6768 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6772 * intel_update_watermarks - update FIFO watermark values based on current modes
6773 * @crtc: the #intel_crtc on which to compute the WM
6775 * Calculate watermark values for the various WM regs based on current mode
6776 * and plane configuration.
6778 * There are several cases to deal with here:
6779 * - normal (i.e. non-self-refresh)
6780 * - self-refresh (SR) mode
6781 * - lines are large relative to FIFO size (buffer can hold up to 2)
6782 * - lines are small relative to FIFO size (buffer can hold more than 2
6783 * lines), so need to account for TLB latency
6785 * The normal calculation is:
6786 * watermark = dotclock * bytes per pixel * latency
6787 * where latency is platform & configuration dependent (we assume pessimal
6790 * The SR calculation is:
6791 * watermark = (trunc(latency/line time)+1) * surface width *
6794 * line time = htotal / dotclock
6795 * surface width = hdisplay for normal plane and 64 for cursor
6796 * and latency is assumed to be high, as above.
6798 * The final value programmed to the register should always be rounded up,
6799 * and include an extra 2 entries to account for clock crossings.
6801 * We don't use the sprite, so we can ignore that. And on Crestline we have
6802 * to set the non-SR watermarks to 8.
6804 void intel_update_watermarks(struct intel_crtc *crtc)
6806 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6808 if (dev_priv->display.update_wm)
6809 dev_priv->display.update_wm(crtc);
6812 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6816 if (!HAS_IPC(dev_priv))
6819 val = I915_READ(DISP_ARB_CTL2);
6821 if (dev_priv->ipc_enabled)
6822 val |= DISP_IPC_ENABLE;
6824 val &= ~DISP_IPC_ENABLE;
6826 I915_WRITE(DISP_ARB_CTL2, val);
6829 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6831 /* Display WA #0477 WaDisableIPC: skl */
6832 if (IS_SKYLAKE(dev_priv))
6835 /* Display WA #1141: SKL:all KBL:all CFL */
6836 if (IS_KABYLAKE(dev_priv) ||
6837 IS_COFFEELAKE(dev_priv) ||
6838 IS_COMETLAKE(dev_priv))
6839 return dev_priv->dram_info.symmetric_memory;
6844 void intel_init_ipc(struct drm_i915_private *dev_priv)
6846 if (!HAS_IPC(dev_priv))
6849 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6851 intel_enable_ipc(dev_priv);
6854 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6857 * On Ibex Peak and Cougar Point, we need to disable clock
6858 * gating for the panel power sequencer or it will fail to
6859 * start up when no ports are active.
6861 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6864 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6868 for_each_pipe(dev_priv, pipe) {
6869 I915_WRITE(DSPCNTR(pipe),
6870 I915_READ(DSPCNTR(pipe)) |
6871 DISPPLANE_TRICKLE_FEED_DISABLE);
6873 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6874 POSTING_READ(DSPSURF(pipe));
6878 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6880 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6884 * WaFbcDisableDpfcClockGating:ilk
6886 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6887 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6888 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6890 I915_WRITE(PCH_3DCGDIS0,
6891 MARIUNIT_CLOCK_GATE_DISABLE |
6892 SVSMUNIT_CLOCK_GATE_DISABLE);
6893 I915_WRITE(PCH_3DCGDIS1,
6894 VFMUNIT_CLOCK_GATE_DISABLE);
6897 * According to the spec the following bits should be set in
6898 * order to enable memory self-refresh
6899 * The bit 22/21 of 0x42004
6900 * The bit 5 of 0x42020
6901 * The bit 15 of 0x45000
6903 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6904 (I915_READ(ILK_DISPLAY_CHICKEN2) |
6905 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6906 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6907 I915_WRITE(DISP_ARB_CTL,
6908 (I915_READ(DISP_ARB_CTL) |
6912 * Based on the document from hardware guys the following bits
6913 * should be set unconditionally in order to enable FBC.
6914 * The bit 22 of 0x42000
6915 * The bit 22 of 0x42004
6916 * The bit 7,8,9 of 0x42020.
6918 if (IS_IRONLAKE_M(dev_priv)) {
6919 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6920 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6921 I915_READ(ILK_DISPLAY_CHICKEN1) |
6923 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6924 I915_READ(ILK_DISPLAY_CHICKEN2) |
6928 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6930 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6931 I915_READ(ILK_DISPLAY_CHICKEN2) |
6932 ILK_ELPIN_409_SELECT);
6934 g4x_disable_trickle_feed(dev_priv);
6936 ibx_init_clock_gating(dev_priv);
6939 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6945 * On Ibex Peak and Cougar Point, we need to disable clock
6946 * gating for the panel power sequencer or it will fail to
6947 * start up when no ports are active.
6949 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6950 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6951 PCH_CPUNIT_CLOCK_GATE_DISABLE);
6952 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6953 DPLS_EDP_PPS_FIX_DIS);
6954 /* The below fixes the weird display corruption, a few pixels shifted
6955 * downward, on (only) LVDS of some HP laptops with IVY.
6957 for_each_pipe(dev_priv, pipe) {
6958 val = I915_READ(TRANS_CHICKEN2(pipe));
6959 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6960 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6961 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6962 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6963 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6964 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6965 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6967 /* WADP0ClockGatingDisable */
6968 for_each_pipe(dev_priv, pipe) {
6969 I915_WRITE(TRANS_CHICKEN1(pipe),
6970 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6974 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6978 tmp = I915_READ(MCH_SSKPD);
6979 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6980 drm_dbg_kms(&dev_priv->drm,
6981 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6985 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6987 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6989 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6991 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6992 I915_READ(ILK_DISPLAY_CHICKEN2) |
6993 ILK_ELPIN_409_SELECT);
6995 I915_WRITE(GEN6_UCGCTL1,
6996 I915_READ(GEN6_UCGCTL1) |
6997 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6998 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7000 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7001 * gating disable must be set. Failure to set it results in
7002 * flickering pixels due to Z write ordering failures after
7003 * some amount of runtime in the Mesa "fire" demo, and Unigine
7004 * Sanctuary and Tropics, and apparently anything else with
7005 * alpha test or pixel discard.
7007 * According to the spec, bit 11 (RCCUNIT) must also be set,
7008 * but we didn't debug actual testcases to find it out.
7010 * WaDisableRCCUnitClockGating:snb
7011 * WaDisableRCPBUnitClockGating:snb
7013 I915_WRITE(GEN6_UCGCTL2,
7014 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7015 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7018 * According to the spec the following bits should be
7019 * set in order to enable memory self-refresh and fbc:
7020 * The bit21 and bit22 of 0x42000
7021 * The bit21 and bit22 of 0x42004
7022 * The bit5 and bit7 of 0x42020
7023 * The bit14 of 0x70180
7024 * The bit14 of 0x71180
7026 * WaFbcAsynchFlipDisableFbcQueue:snb
7028 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7029 I915_READ(ILK_DISPLAY_CHICKEN1) |
7030 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7031 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7032 I915_READ(ILK_DISPLAY_CHICKEN2) |
7033 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7034 I915_WRITE(ILK_DSPCLK_GATE_D,
7035 I915_READ(ILK_DSPCLK_GATE_D) |
7036 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7037 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7039 g4x_disable_trickle_feed(dev_priv);
7041 cpt_init_clock_gating(dev_priv);
7043 gen6_check_mch_setup(dev_priv);
7046 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7049 * TODO: this bit should only be enabled when really needed, then
7050 * disabled when not needed anymore in order to save power.
7052 if (HAS_PCH_LPT_LP(dev_priv))
7053 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7054 I915_READ(SOUTH_DSPCLK_GATE_D) |
7055 PCH_LP_PARTITION_LEVEL_DISABLE);
7057 /* WADPOClockGatingDisable:hsw */
7058 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7059 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7060 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7063 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7065 if (HAS_PCH_LPT_LP(dev_priv)) {
7066 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7068 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7069 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7073 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7074 int general_prio_credits,
7075 int high_prio_credits)
7080 /* WaTempDisableDOPClkGating:bdw */
7081 misccpctl = I915_READ(GEN7_MISCCPCTL);
7082 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7084 val = I915_READ(GEN8_L3SQCREG1);
7085 val &= ~L3_PRIO_CREDITS_MASK;
7086 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7087 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7088 I915_WRITE(GEN8_L3SQCREG1, val);
7091 * Wait at least 100 clocks before re-enabling clock gating.
7092 * See the definition of L3SQCREG1 in BSpec.
7094 POSTING_READ(GEN8_L3SQCREG1);
7096 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7099 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7101 /* Wa_1409120013:icl,ehl */
7102 I915_WRITE(ILK_DPFC_CHICKEN,
7103 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7105 /* This is not an Wa. Enable to reduce Sampler power */
7106 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7107 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7109 /*Wa_14010594013:icl, ehl */
7110 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7111 0, CNL_DELAY_PMRSP);
7114 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7116 u32 vd_pg_enable = 0;
7119 /* Wa_1409120013:tgl */
7120 I915_WRITE(ILK_DPFC_CHICKEN,
7121 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7123 /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7124 for (i = 0; i < I915_MAX_VCS; i++) {
7125 if (HAS_ENGINE(&dev_priv->gt, _VCS(i)))
7126 vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7127 VDN_MFX_POWERGATE_ENABLE(i);
7130 I915_WRITE(POWERGATE_ENABLE,
7131 I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7133 /* Wa_1409825376:tgl (pre-prod)*/
7134 if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
7135 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7136 TGL_VRH_GATING_DIS);
7138 /* Wa_14011059788:tgl */
7139 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7143 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7145 if (!HAS_PCH_CNP(dev_priv))
7148 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7149 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7150 CNP_PWM_CGE_GATING_DISABLE);
7153 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7156 cnp_init_clock_gating(dev_priv);
7158 /* This is not an Wa. Enable for better image quality */
7159 I915_WRITE(_3D_CHICKEN3,
7160 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7162 /* WaEnableChickenDCPR:cnl */
7163 I915_WRITE(GEN8_CHICKEN_DCPR_1,
7164 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7166 /* WaFbcWakeMemOn:cnl */
7167 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7168 DISP_FBC_MEMORY_WAKE);
7170 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7171 /* ReadHitWriteOnlyDisable:cnl */
7172 val |= RCCUNIT_CLKGATE_DIS;
7173 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7175 /* Wa_2201832410:cnl */
7176 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7177 val |= GWUNIT_CLKGATE_DIS;
7178 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7180 /* WaDisableVFclkgate:cnl */
7181 /* WaVFUnitClockGatingDisable:cnl */
7182 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7183 val |= VFUNIT_CLKGATE_DIS;
7184 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7187 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7189 cnp_init_clock_gating(dev_priv);
7190 gen9_init_clock_gating(dev_priv);
7192 /* WaFbcNukeOnHostModify:cfl */
7193 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7194 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7197 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7199 gen9_init_clock_gating(dev_priv);
7201 /* WaDisableSDEUnitClockGating:kbl */
7202 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7203 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7204 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7206 /* WaDisableGamClockGating:kbl */
7207 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7208 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7209 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7211 /* WaFbcNukeOnHostModify:kbl */
7212 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7213 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7216 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7218 gen9_init_clock_gating(dev_priv);
7220 /* WAC6entrylatency:skl */
7221 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7222 FBC_LLC_FULLY_OPEN);
7224 /* WaFbcNukeOnHostModify:skl */
7225 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7226 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7229 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7233 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7234 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7235 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7238 /* WaSwitchSolVfFArbitrationPriority:bdw */
7239 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7241 /* WaPsrDPAMaskVBlankInSRD:bdw */
7242 I915_WRITE(CHICKEN_PAR1_1,
7243 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7245 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7246 for_each_pipe(dev_priv, pipe) {
7247 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7248 I915_READ(CHICKEN_PIPESL_1(pipe)) |
7249 BDW_DPRS_MASK_VBLANK_SRD);
7252 /* WaVSRefCountFullforceMissDisable:bdw */
7253 /* WaDSRefCountFullforceMissDisable:bdw */
7254 I915_WRITE(GEN7_FF_THREAD_MODE,
7255 I915_READ(GEN7_FF_THREAD_MODE) &
7256 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7258 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7259 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7261 /* WaDisableSDEUnitClockGating:bdw */
7262 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7263 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7265 /* WaProgramL3SqcReg1Default:bdw */
7266 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7268 /* WaKVMNotificationOnConfigChange:bdw */
7269 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7270 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7272 lpt_init_clock_gating(dev_priv);
7274 /* WaDisableDopClockGating:bdw
7276 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7279 I915_WRITE(GEN6_UCGCTL1,
7280 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7283 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7285 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7286 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7287 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7290 /* This is required by WaCatErrorRejectionIssue:hsw */
7291 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7292 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7293 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7295 /* WaSwitchSolVfFArbitrationPriority:hsw */
7296 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7298 lpt_init_clock_gating(dev_priv);
7301 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7305 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7307 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7308 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7309 I915_READ(ILK_DISPLAY_CHICKEN1) |
7312 /* WaDisableBackToBackFlipFix:ivb */
7313 I915_WRITE(IVB_CHICKEN3,
7314 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7315 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7317 if (IS_IVB_GT1(dev_priv))
7318 I915_WRITE(GEN7_ROW_CHICKEN2,
7319 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7321 /* must write both registers */
7322 I915_WRITE(GEN7_ROW_CHICKEN2,
7323 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7324 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7325 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7329 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7330 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7332 I915_WRITE(GEN6_UCGCTL2,
7333 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7335 /* This is required by WaCatErrorRejectionIssue:ivb */
7336 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7337 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7338 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7340 g4x_disable_trickle_feed(dev_priv);
7342 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7343 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7344 snpcr |= GEN6_MBC_SNPCR_MED;
7345 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7347 if (!HAS_PCH_NOP(dev_priv))
7348 cpt_init_clock_gating(dev_priv);
7350 gen6_check_mch_setup(dev_priv);
7353 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7355 /* WaDisableBackToBackFlipFix:vlv */
7356 I915_WRITE(IVB_CHICKEN3,
7357 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7358 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7360 /* WaDisableDopClockGating:vlv */
7361 I915_WRITE(GEN7_ROW_CHICKEN2,
7362 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7364 /* This is required by WaCatErrorRejectionIssue:vlv */
7365 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7366 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7367 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7370 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7371 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7373 I915_WRITE(GEN6_UCGCTL2,
7374 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7376 /* WaDisableL3Bank2xClockGate:vlv
7377 * Disabling L3 clock gating- MMIO 940c[25] = 1
7378 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7379 I915_WRITE(GEN7_UCGCTL4,
7380 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7383 * WaDisableVLVClockGating_VBIIssue:vlv
7384 * Disable clock gating on th GCFG unit to prevent a delay
7385 * in the reporting of vblank events.
7387 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7390 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7392 /* WaVSRefCountFullforceMissDisable:chv */
7393 /* WaDSRefCountFullforceMissDisable:chv */
7394 I915_WRITE(GEN7_FF_THREAD_MODE,
7395 I915_READ(GEN7_FF_THREAD_MODE) &
7396 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7398 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7399 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7400 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7402 /* WaDisableCSUnitClockGating:chv */
7403 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7404 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7406 /* WaDisableSDEUnitClockGating:chv */
7407 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7408 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7411 * WaProgramL3SqcReg1Default:chv
7412 * See gfxspecs/Related Documents/Performance Guide/
7413 * LSQC Setting Recommendations.
7415 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7418 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7422 I915_WRITE(RENCLK_GATE_D1, 0);
7423 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7424 GS_UNIT_CLOCK_GATE_DISABLE |
7425 CL_UNIT_CLOCK_GATE_DISABLE);
7426 I915_WRITE(RAMCLK_GATE_D, 0);
7427 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7428 OVRUNIT_CLOCK_GATE_DISABLE |
7429 OVCUNIT_CLOCK_GATE_DISABLE;
7430 if (IS_GM45(dev_priv))
7431 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7432 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7434 g4x_disable_trickle_feed(dev_priv);
7437 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7439 struct intel_uncore *uncore = &dev_priv->uncore;
7441 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7442 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7443 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7444 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7445 intel_uncore_write16(uncore, DEUC, 0);
7446 intel_uncore_write(uncore,
7448 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7451 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7453 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7454 I965_RCC_CLOCK_GATE_DISABLE |
7455 I965_RCPB_CLOCK_GATE_DISABLE |
7456 I965_ISC_CLOCK_GATE_DISABLE |
7457 I965_FBC_CLOCK_GATE_DISABLE);
7458 I915_WRITE(RENCLK_GATE_D2, 0);
7459 I915_WRITE(MI_ARB_STATE,
7460 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7463 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7465 u32 dstate = I915_READ(D_STATE);
7467 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7468 DSTATE_DOT_CLOCK_GATING;
7469 I915_WRITE(D_STATE, dstate);
7471 if (IS_PINEVIEW(dev_priv))
7472 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7474 /* IIR "flip pending" means done if this bit is set */
7475 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7477 /* interrupts should cause a wake up from C3 */
7478 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7480 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7481 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7483 I915_WRITE(MI_ARB_STATE,
7484 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7487 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7489 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7491 /* interrupts should cause a wake up from C3 */
7492 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7493 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7495 I915_WRITE(MEM_MODE,
7496 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7499 * Have FBC ignore 3D activity since we use software
7500 * render tracking, and otherwise a pure 3D workload
7501 * (even if it just renders a single frame and then does
7502 * abosultely nothing) would not allow FBC to recompress
7503 * until a 2D blit occurs.
7506 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7509 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7511 I915_WRITE(MEM_MODE,
7512 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7513 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7516 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7518 dev_priv->display.init_clock_gating(dev_priv);
7521 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7523 if (HAS_PCH_LPT(dev_priv))
7524 lpt_suspend_hw(dev_priv);
7527 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7529 drm_dbg_kms(&dev_priv->drm,
7530 "No clock gating settings or workarounds applied.\n");
7534 * intel_init_clock_gating_hooks - setup the clock gating hooks
7535 * @dev_priv: device private
7537 * Setup the hooks that configure which clocks of a given platform can be
7538 * gated and also apply various GT and display specific workarounds for these
7539 * platforms. Note that some GT specific workarounds are applied separately
7540 * when GPU contexts or batchbuffers start their execution.
7542 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7544 if (IS_GEN(dev_priv, 12))
7545 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7546 else if (IS_GEN(dev_priv, 11))
7547 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7548 else if (IS_CANNONLAKE(dev_priv))
7549 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7550 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7551 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7552 else if (IS_SKYLAKE(dev_priv))
7553 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7554 else if (IS_KABYLAKE(dev_priv))
7555 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7556 else if (IS_BROXTON(dev_priv))
7557 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7558 else if (IS_GEMINILAKE(dev_priv))
7559 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7560 else if (IS_BROADWELL(dev_priv))
7561 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7562 else if (IS_CHERRYVIEW(dev_priv))
7563 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7564 else if (IS_HASWELL(dev_priv))
7565 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7566 else if (IS_IVYBRIDGE(dev_priv))
7567 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7568 else if (IS_VALLEYVIEW(dev_priv))
7569 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7570 else if (IS_GEN(dev_priv, 6))
7571 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7572 else if (IS_GEN(dev_priv, 5))
7573 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7574 else if (IS_G4X(dev_priv))
7575 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7576 else if (IS_I965GM(dev_priv))
7577 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7578 else if (IS_I965G(dev_priv))
7579 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7580 else if (IS_GEN(dev_priv, 3))
7581 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7582 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7583 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7584 else if (IS_GEN(dev_priv, 2))
7585 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7587 MISSING_CASE(INTEL_DEVID(dev_priv));
7588 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7592 /* Set up chip specific power management-related functions */
7593 void intel_init_pm(struct drm_i915_private *dev_priv)
7596 if (IS_PINEVIEW(dev_priv))
7597 pnv_get_mem_freq(dev_priv);
7598 else if (IS_GEN(dev_priv, 5))
7599 ilk_get_mem_freq(dev_priv);
7601 if (intel_has_sagv(dev_priv))
7602 skl_setup_sagv_block_time(dev_priv);
7604 /* For FIFO watermark updates */
7605 if (INTEL_GEN(dev_priv) >= 9) {
7606 skl_setup_wm_latency(dev_priv);
7607 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7608 } else if (HAS_PCH_SPLIT(dev_priv)) {
7609 ilk_setup_wm_latency(dev_priv);
7611 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7612 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7613 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7614 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7615 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7616 dev_priv->display.compute_intermediate_wm =
7617 ilk_compute_intermediate_wm;
7618 dev_priv->display.initial_watermarks =
7619 ilk_initial_watermarks;
7620 dev_priv->display.optimize_watermarks =
7621 ilk_optimize_watermarks;
7623 drm_dbg_kms(&dev_priv->drm,
7624 "Failed to read display plane latency. "
7627 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7628 vlv_setup_wm_latency(dev_priv);
7629 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7630 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7631 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7632 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7633 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7634 } else if (IS_G4X(dev_priv)) {
7635 g4x_setup_wm_latency(dev_priv);
7636 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7637 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7638 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7639 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7640 } else if (IS_PINEVIEW(dev_priv)) {
7641 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7644 dev_priv->mem_freq)) {
7645 drm_info(&dev_priv->drm,
7646 "failed to find known CxSR latency "
7647 "(found ddr%s fsb freq %d, mem freq %d), "
7649 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7650 dev_priv->fsb_freq, dev_priv->mem_freq);
7651 /* Disable CxSR and never update its watermark again */
7652 intel_set_memory_cxsr(dev_priv, false);
7653 dev_priv->display.update_wm = NULL;
7655 dev_priv->display.update_wm = pnv_update_wm;
7656 } else if (IS_GEN(dev_priv, 4)) {
7657 dev_priv->display.update_wm = i965_update_wm;
7658 } else if (IS_GEN(dev_priv, 3)) {
7659 dev_priv->display.update_wm = i9xx_update_wm;
7660 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7661 } else if (IS_GEN(dev_priv, 2)) {
7662 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7663 dev_priv->display.update_wm = i845_update_wm;
7664 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7666 dev_priv->display.update_wm = i9xx_update_wm;
7667 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7670 drm_err(&dev_priv->drm,
7671 "unexpected fall-through in %s\n", __func__);
7675 void intel_pm_setup(struct drm_i915_private *dev_priv)
7677 dev_priv->runtime_pm.suspended = false;
7678 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7681 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7683 struct intel_dbuf_state *dbuf_state;
7685 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7689 return &dbuf_state->base;
7692 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7693 struct intel_global_state *state)
7698 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7699 .atomic_duplicate_state = intel_dbuf_duplicate_state,
7700 .atomic_destroy_state = intel_dbuf_destroy_state,
7703 struct intel_dbuf_state *
7704 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7706 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7707 struct intel_global_state *dbuf_state;
7709 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7710 if (IS_ERR(dbuf_state))
7711 return ERR_CAST(dbuf_state);
7713 return to_intel_dbuf_state(dbuf_state);
7716 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7718 struct intel_dbuf_state *dbuf_state;
7720 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7724 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7725 &dbuf_state->base, &intel_dbuf_funcs);
7730 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7732 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7733 const struct intel_dbuf_state *new_dbuf_state =
7734 intel_atomic_get_new_dbuf_state(state);
7735 const struct intel_dbuf_state *old_dbuf_state =
7736 intel_atomic_get_old_dbuf_state(state);
7738 if (!new_dbuf_state ||
7739 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7742 WARN_ON(!new_dbuf_state->base.changed);
7744 gen9_dbuf_slices_update(dev_priv,
7745 old_dbuf_state->enabled_slices |
7746 new_dbuf_state->enabled_slices);
7749 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7751 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7752 const struct intel_dbuf_state *new_dbuf_state =
7753 intel_atomic_get_new_dbuf_state(state);
7754 const struct intel_dbuf_state *old_dbuf_state =
7755 intel_atomic_get_old_dbuf_state(state);
7757 if (!new_dbuf_state ||
7758 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7761 WARN_ON(!new_dbuf_state->base.changed);
7763 gen9_dbuf_slices_update(dev_priv,
7764 new_dbuf_state->enabled_slices);