drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include <drm/drm_atomic_helper.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include "display/intel_atomic.h"
  36 #include "display/intel_display_types.h"
  37 #include "display/intel_fbc.h"
  38 #include "display/intel_sprite.h"
  39
  40 #include "gt/intel_llc.h"
  41
  42 #include "i915_drv.h"
  43 #include "i915_irq.h"
  44 #include "i915_trace.h"
  45 #include "intel_pm.h"
  46 #include "intel_sideband.h"
  47 #include "../../../platform/x86/intel_ips.h"
  48
  49 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  50 {
  51         if (HAS_LLC(dev_priv)) {
  52                 /*
  53                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  54                  * Display WA #0390: skl,kbl
  55                  *
  56                  * Must match Sampler, Pixel Back End, and Media. See
  57                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  58                  */
  59                 I915_WRITE(CHICKEN_PAR1_1,
  60                            I915_READ(CHICKEN_PAR1_1) |
  61                            SKL_DE_COMPRESSED_HASH_MODE);
  62         }
  63
  64         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  65         I915_WRITE(CHICKEN_PAR1_1,
  66                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  67
  68         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  69         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  70                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  71
  72         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  73         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  74         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  75                    DISP_FBC_WM_DIS |
  76                    DISP_FBC_MEMORY_WAKE);
  77
  78         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
  79         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
  80                    ILK_DPFC_DISABLE_DUMMY0);
  81
  82         if (IS_SKYLAKE(dev_priv)) {
  83                 /* WaDisableDopClockGating */
  84                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
  85                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  86         }
  87 }
  88
  89 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  90 {
  91         gen9_init_clock_gating(dev_priv);
  92
  93         /* WaDisableSDEUnitClockGating:bxt */
  94         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
  95                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  96
  97         /*
  98          * FIXME:
  99          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 100          */
 101         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 102                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 103
 104         /*
 105          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 106          * to stay fully on.
 107          */
 108         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 109                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 110
 111         /*
 112          * Lower the display internal timeout.
 113          * This is needed to avoid any hard hangs when DSI port PLL
 114          * is off and a MMIO access is attempted by any privilege
 115          * application, using batch buffers or any other means.
 116          */
 117         I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 118 }
 119
 120 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 121 {
 122         gen9_init_clock_gating(dev_priv);
 123
 124         /*
 125          * WaDisablePWMClockGating:glk
 126          * Backlight PWM may stop in the asserted state, causing backlight
 127          * to stay fully on.
 128          */
 129         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 130                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 131
 132         /* WaDDIIOTimeout:glk */
 133         if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
 134                 u32 val = I915_READ(CHICKEN_MISC_2);
 135                 val &= ~(GLK_CL0_PWR_DOWN |
 136                          GLK_CL1_PWR_DOWN |
 137                          GLK_CL2_PWR_DOWN);
 138                 I915_WRITE(CHICKEN_MISC_2, val);
 139         }
 140
 141 }
 142
 143 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
 144 {
 145         u32 tmp;
 146
 147         tmp = I915_READ(CLKCFG);
 148
 149         switch (tmp & CLKCFG_FSB_MASK) {
 150         case CLKCFG_FSB_533:
 151                 dev_priv->fsb_freq = 533; /* 133*4 */
 152                 break;
 153         case CLKCFG_FSB_800:
 154                 dev_priv->fsb_freq = 800; /* 200*4 */
 155                 break;
 156         case CLKCFG_FSB_667:
 157                 dev_priv->fsb_freq =  667; /* 167*4 */
 158                 break;
 159         case CLKCFG_FSB_400:
 160                 dev_priv->fsb_freq = 400; /* 100*4 */
 161                 break;
 162         }
 163
 164         switch (tmp & CLKCFG_MEM_MASK) {
 165         case CLKCFG_MEM_533:
 166                 dev_priv->mem_freq = 533;
 167                 break;
 168         case CLKCFG_MEM_667:
 169                 dev_priv->mem_freq = 667;
 170                 break;
 171         case CLKCFG_MEM_800:
 172                 dev_priv->mem_freq = 800;
 173                 break;
 174         }
 175
 176         /* detect pineview DDR3 setting */
 177         tmp = I915_READ(CSHRDDR3CTL);
 178         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 179 }
 180
 181 static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
 182 {
 183         u16 ddrpll, csipll;
 184
 185         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 186         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 187
 188         switch (ddrpll & 0xff) {
 189         case 0xc:
 190                 dev_priv->mem_freq = 800;
 191                 break;
 192         case 0x10:
 193                 dev_priv->mem_freq = 1066;
 194                 break;
 195         case 0x14:
 196                 dev_priv->mem_freq = 1333;
 197                 break;
 198         case 0x18:
 199                 dev_priv->mem_freq = 1600;
 200                 break;
 201         default:
 202                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 203                                  ddrpll & 0xff);
 204                 dev_priv->mem_freq = 0;
 205                 break;
 206         }
 207
 208         switch (csipll & 0x3ff) {
 209         case 0x00c:
 210                 dev_priv->fsb_freq = 3200;
 211                 break;
 212         case 0x00e:
 213                 dev_priv->fsb_freq = 3733;
 214                 break;
 215         case 0x010:
 216                 dev_priv->fsb_freq = 4266;
 217                 break;
 218         case 0x012:
 219                 dev_priv->fsb_freq = 4800;
 220                 break;
 221         case 0x014:
 222                 dev_priv->fsb_freq = 5333;
 223                 break;
 224         case 0x016:
 225                 dev_priv->fsb_freq = 5866;
 226                 break;
 227         case 0x018:
 228                 dev_priv->fsb_freq = 6400;
 229                 break;
 230         default:
 231                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 232                                  csipll & 0x3ff);
 233                 dev_priv->fsb_freq = 0;
 234                 break;
 235         }
 236 }
 237
 238 static const struct cxsr_latency cxsr_latency_table[] = {
 239         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 240         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 241         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 242         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 243         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 244
 245         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 246         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 247         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 248         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 249         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 250
 251         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 252         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 253         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 254         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 255         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 256
 257         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 258         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 259         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 260         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 261         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 262
 263         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 264         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 265         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 266         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 267         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 268
 269         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 270         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 271         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 272         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 273         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 274 };
 275
 276 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 277                                                          bool is_ddr3,
 278                                                          int fsb,
 279                                                          int mem)
 280 {
 281         const struct cxsr_latency *latency;
 282         int i;
 283
 284         if (fsb == 0 || mem == 0)
 285                 return NULL;
 286
 287         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 288                 latency = &cxsr_latency_table[i];
 289                 if (is_desktop == latency->is_desktop &&
 290                     is_ddr3 == latency->is_ddr3 &&
 291                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 292                         return latency;
 293         }
 294
 295         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 296
 297         return NULL;
 298 }
 299
 300 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 301 {
 302         u32 val;
 303
 304         vlv_punit_get(dev_priv);
 305
 306         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 307         if (enable)
 308                 val &= ~FORCE_DDR_HIGH_FREQ;
 309         else
 310                 val |= FORCE_DDR_HIGH_FREQ;
 311         val &= ~FORCE_DDR_LOW_FREQ;
 312         val |= FORCE_DDR_FREQ_REQ_ACK;
 313         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 314
 315         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 316                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 317                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 318
 319         vlv_punit_put(dev_priv);
 320 }
 321
 322 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 323 {
 324         u32 val;
 325
 326         vlv_punit_get(dev_priv);
 327
 328         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 329         if (enable)
 330                 val |= DSP_MAXFIFO_PM5_ENABLE;
 331         else
 332                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 333         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 334
 335         vlv_punit_put(dev_priv);
 336 }
 337
 338 #define FW_WM(value, plane) \
 339         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 340
 341 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 342 {
 343         bool was_enabled;
 344         u32 val;
 345
 346         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 347                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 348                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 349                 POSTING_READ(FW_BLC_SELF_VLV);
 350         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 351                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 352                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 353                 POSTING_READ(FW_BLC_SELF);
 354         } else if (IS_PINEVIEW(dev_priv)) {
 355                 val = I915_READ(DSPFW3);
 356                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 357                 if (enable)
 358                         val |= PINEVIEW_SELF_REFRESH_EN;
 359                 else
 360                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 361                 I915_WRITE(DSPFW3, val);
 362                 POSTING_READ(DSPFW3);
 363         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 364                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 365                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 366                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 367                 I915_WRITE(FW_BLC_SELF, val);
 368                 POSTING_READ(FW_BLC_SELF);
 369         } else if (IS_I915GM(dev_priv)) {
 370                 /*
 371                  * FIXME can't find a bit like this for 915G, and
 372                  * and yet it does have the related watermark in
 373                  * FW_BLC_SELF. What's going on?
 374                  */
 375                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 376                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 377                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 378                 I915_WRITE(INSTPM, val);
 379                 POSTING_READ(INSTPM);
 380         } else {
 381                 return false;
 382         }
 383
 384         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 385
 386         DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
 387                       enableddisabled(enable),
 388                       enableddisabled(was_enabled));
 389
 390         return was_enabled;
 391 }
 392
 393 /**
 394  * intel_set_memory_cxsr - Configure CxSR state
 395  * @dev_priv: i915 device
 396  * @enable: Allow vs. disallow CxSR
 397  *
 398  * Allow or disallow the system to enter a special CxSR
 399  * (C-state self refresh) state. What typically happens in CxSR mode
 400  * is that several display FIFOs may get combined into a single larger
 401  * FIFO for a particular plane (so called max FIFO mode) to allow the
 402  * system to defer memory fetches longer, and the memory will enter
 403  * self refresh.
 404  *
 405  * Note that enabling CxSR does not guarantee that the system enter
 406  * this special mode, nor does it guarantee that the system stays
 407  * in that mode once entered. So this just allows/disallows the system
 408  * to autonomously utilize the CxSR mode. Other factors such as core
 409  * C-states will affect when/if the system actually enters/exits the
 410  * CxSR mode.
 411  *
 412  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 413  * and the system is free to enter/exit memory self refresh at any time
 414  * even when the use of CxSR has been disallowed.
 415  *
 416  * While the system is actually in the CxSR/max FIFO mode, some plane
 417  * control registers will not get latched on vblank. Thus in order to
 418  * guarantee the system will respond to changes in the plane registers
 419  * we must always disallow CxSR prior to making changes to those registers.
 420  * Unfortunately the system will re-evaluate the CxSR conditions at
 421  * frame start which happens after vblank start (which is when the plane
 422  * registers would get latched), so we can't proceed with the plane update
 423  * during the same frame where we disallowed CxSR.
 424  *
 425  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 426  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 427  * the hardware w.r.t. HPLL SR when writing to plane registers.
 428  * Disallowing just CxSR is sufficient.
 429  */
 430 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 431 {
 432         bool ret;
 433
 434         mutex_lock(&dev_priv->wm.wm_mutex);
 435         ret = _intel_set_memory_cxsr(dev_priv, enable);
 436         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 437                 dev_priv->wm.vlv.cxsr = enable;
 438         else if (IS_G4X(dev_priv))
 439                 dev_priv->wm.g4x.cxsr = enable;
 440         mutex_unlock(&dev_priv->wm.wm_mutex);
 441
 442         return ret;
 443 }
 444
 445 /*
 446  * Latency for FIFO fetches is dependent on several factors:
 447  *   - memory configuration (speed, channels)
 448  *   - chipset
 449  *   - current MCH state
 450  * It can be fairly high in some situations, so here we assume a fairly
 451  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 452  * set this value too high, the FIFO will fetch frequently to stay full)
 453  * and power consumption (set it too low to save power and we might see
 454  * FIFO underruns and display "flicker").
 455  *
 456  * A value of 5us seems to be a good balance; safe for very low end
 457  * platforms but not overly aggressive on lower latency configs.
 458  */
 459 static const int pessimal_latency_ns = 5000;
 460
 461 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 462         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 463
 464 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 465 {
 466         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 467         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 468         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 469         enum pipe pipe = crtc->pipe;
 470         int sprite0_start, sprite1_start;
 471
 472         switch (pipe) {
 473                 u32 dsparb, dsparb2, dsparb3;
 474         case PIPE_A:
 475                 dsparb = I915_READ(DSPARB);
 476                 dsparb2 = I915_READ(DSPARB2);
 477                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 478                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 479                 break;
 480         case PIPE_B:
 481                 dsparb = I915_READ(DSPARB);
 482                 dsparb2 = I915_READ(DSPARB2);
 483                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 484                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 485                 break;
 486         case PIPE_C:
 487                 dsparb2 = I915_READ(DSPARB2);
 488                 dsparb3 = I915_READ(DSPARB3);
 489                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 490                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 491                 break;
 492         default:
 493                 MISSING_CASE(pipe);
 494                 return;
 495         }
 496
 497         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 498         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 499         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 500         fifo_state->plane[PLANE_CURSOR] = 63;
 501 }
 502
 503 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 504                               enum i9xx_plane_id i9xx_plane)
 505 {
 506         u32 dsparb = I915_READ(DSPARB);
 507         int size;
 508
 509         size = dsparb & 0x7f;
 510         if (i9xx_plane == PLANE_B)
 511                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 512
 513         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 514                       dsparb, plane_name(i9xx_plane), size);
 515
 516         return size;
 517 }
 518
 519 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 520                               enum i9xx_plane_id i9xx_plane)
 521 {
 522         u32 dsparb = I915_READ(DSPARB);
 523         int size;
 524
 525         size = dsparb & 0x1ff;
 526         if (i9xx_plane == PLANE_B)
 527                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 528         size >>= 1; /* Convert to cachelines */
 529
 530         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 531                       dsparb, plane_name(i9xx_plane), size);
 532
 533         return size;
 534 }
 535
 536 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 537                               enum i9xx_plane_id i9xx_plane)
 538 {
 539         u32 dsparb = I915_READ(DSPARB);
 540         int size;
 541
 542         size = dsparb & 0x7f;
 543         size >>= 2; /* Convert to cachelines */
 544
 545         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 546                       dsparb, plane_name(i9xx_plane), size);
 547
 548         return size;
 549 }
 550
 551 /* Pineview has different values for various configs */
 552 static const struct intel_watermark_params pineview_display_wm = {
 553         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 554         .max_wm = PINEVIEW_MAX_WM,
 555         .default_wm = PINEVIEW_DFT_WM,
 556         .guard_size = PINEVIEW_GUARD_WM,
 557         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 558 };
 559 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 560         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 561         .max_wm = PINEVIEW_MAX_WM,
 562         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 563         .guard_size = PINEVIEW_GUARD_WM,
 564         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 565 };
 566 static const struct intel_watermark_params pineview_cursor_wm = {
 567         .fifo_size = PINEVIEW_CURSOR_FIFO,
 568         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 569         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 570         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 571         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 572 };
 573 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 574         .fifo_size = PINEVIEW_CURSOR_FIFO,
 575         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 576         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 577         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 578         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 579 };
 580 static const struct intel_watermark_params i965_cursor_wm_info = {
 581         .fifo_size = I965_CURSOR_FIFO,
 582         .max_wm = I965_CURSOR_MAX_WM,
 583         .default_wm = I965_CURSOR_DFT_WM,
 584         .guard_size = 2,
 585         .cacheline_size = I915_FIFO_LINE_SIZE,
 586 };
 587 static const struct intel_watermark_params i945_wm_info = {
 588         .fifo_size = I945_FIFO_SIZE,
 589         .max_wm = I915_MAX_WM,
 590         .default_wm = 1,
 591         .guard_size = 2,
 592         .cacheline_size = I915_FIFO_LINE_SIZE,
 593 };
 594 static const struct intel_watermark_params i915_wm_info = {
 595         .fifo_size = I915_FIFO_SIZE,
 596         .max_wm = I915_MAX_WM,
 597         .default_wm = 1,
 598         .guard_size = 2,
 599         .cacheline_size = I915_FIFO_LINE_SIZE,
 600 };
 601 static const struct intel_watermark_params i830_a_wm_info = {
 602         .fifo_size = I855GM_FIFO_SIZE,
 603         .max_wm = I915_MAX_WM,
 604         .default_wm = 1,
 605         .guard_size = 2,
 606         .cacheline_size = I830_FIFO_LINE_SIZE,
 607 };
 608 static const struct intel_watermark_params i830_bc_wm_info = {
 609         .fifo_size = I855GM_FIFO_SIZE,
 610         .max_wm = I915_MAX_WM/2,
 611         .default_wm = 1,
 612         .guard_size = 2,
 613         .cacheline_size = I830_FIFO_LINE_SIZE,
 614 };
 615 static const struct intel_watermark_params i845_wm_info = {
 616         .fifo_size = I830_FIFO_SIZE,
 617         .max_wm = I915_MAX_WM,
 618         .default_wm = 1,
 619         .guard_size = 2,
 620         .cacheline_size = I830_FIFO_LINE_SIZE,
 621 };
 622
 623 /**
 624  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 625  * @pixel_rate: Pipe pixel rate in kHz
 626  * @cpp: Plane bytes per pixel
 627  * @latency: Memory wakeup latency in 0.1us units
 628  *
 629  * Compute the watermark using the method 1 or "small buffer"
 630  * formula. The caller may additonally add extra cachelines
 631  * to account for TLB misses and clock crossings.
 632  *
 633  * This method is concerned with the short term drain rate
 634  * of the FIFO, ie. it does not account for blanking periods
 635  * which would effectively reduce the average drain rate across
 636  * a longer period. The name "small" refers to the fact the
 637  * FIFO is relatively small compared to the amount of data
 638  * fetched.
 639  *
 640  * The FIFO level vs. time graph might look something like:
 641  *
 642  *   |\   |\
 643  *   | \  | \
 644  * __---__---__ (- plane active, _ blanking)
 645  * -> time
 646  *
 647  * or perhaps like this:
 648  *
 649  *   |\|\  |\|\
 650  * __----__----__ (- plane active, _ blanking)
 651  * -> time
 652  *
 653  * Returns:
 654  * The watermark in bytes
 655  */
 656 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 657                                      unsigned int cpp,
 658                                      unsigned int latency)
 659 {
 660         u64 ret;
 661
 662         ret = mul_u32_u32(pixel_rate, cpp * latency);
 663         ret = DIV_ROUND_UP_ULL(ret, 10000);
 664
 665         return ret;
 666 }
 667
 668 /**
 669  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 670  * @pixel_rate: Pipe pixel rate in kHz
 671  * @htotal: Pipe horizontal total
 672  * @width: Plane width in pixels
 673  * @cpp: Plane bytes per pixel
 674  * @latency: Memory wakeup latency in 0.1us units
 675  *
 676  * Compute the watermark using the method 2 or "large buffer"
 677  * formula. The caller may additonally add extra cachelines
 678  * to account for TLB misses and clock crossings.
 679  *
 680  * This method is concerned with the long term drain rate
 681  * of the FIFO, ie. it does account for blanking periods
 682  * which effectively reduce the average drain rate across
 683  * a longer period. The name "large" refers to the fact the
 684  * FIFO is relatively large compared to the amount of data
 685  * fetched.
 686  *
 687  * The FIFO level vs. time graph might look something like:
 688  *
 689  *    |\___       |\___
 690  *    |    \___   |    \___
 691  *    |        \  |        \
 692  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 693  * -> time
 694  *
 695  * Returns:
 696  * The watermark in bytes
 697  */
 698 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 699                                      unsigned int htotal,
 700                                      unsigned int width,
 701                                      unsigned int cpp,
 702                                      unsigned int latency)
 703 {
 704         unsigned int ret;
 705
 706         /*
 707          * FIXME remove once all users are computing
 708          * watermarks in the correct place.
 709          */
 710         if (WARN_ON_ONCE(htotal == 0))
 711                 htotal = 1;
 712
 713         ret = (latency * pixel_rate) / (htotal * 10000);
 714         ret = (ret + 1) * width * cpp;
 715
 716         return ret;
 717 }
 718
 719 /**
 720  * intel_calculate_wm - calculate watermark level
 721  * @pixel_rate: pixel clock
 722  * @wm: chip FIFO params
 723  * @fifo_size: size of the FIFO buffer
 724  * @cpp: bytes per pixel
 725  * @latency_ns: memory latency for the platform
 726  *
 727  * Calculate the watermark level (the level at which the display plane will
 728  * start fetching from memory again).  Each chip has a different display
 729  * FIFO size and allocation, so the caller needs to figure that out and pass
 730  * in the correct intel_watermark_params structure.
 731  *
 732  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 733  * on the pixel size.  When it reaches the watermark level, it'll start
 734  * fetching FIFO line sized based chunks from memory until the FIFO fills
 735  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 736  * will occur, and a display engine hang could result.
 737  */
 738 static unsigned int intel_calculate_wm(int pixel_rate,
 739                                        const struct intel_watermark_params *wm,
 740                                        int fifo_size, int cpp,
 741                                        unsigned int latency_ns)
 742 {
 743         int entries, wm_size;
 744
 745         /*
 746          * Note: we need to make sure we don't overflow for various clock &
 747          * latency values.
 748          * clocks go from a few thousand to several hundred thousand.
 749          * latency is usually a few thousand
 750          */
 751         entries = intel_wm_method1(pixel_rate, cpp,
 752                                    latency_ns / 100);
 753         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 754                 wm->guard_size;
 755         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 756
 757         wm_size = fifo_size - entries;
 758         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 759
 760         /* Don't promote wm_size to unsigned... */
 761         if (wm_size > wm->max_wm)
 762                 wm_size = wm->max_wm;
 763         if (wm_size <= 0)
 764                 wm_size = wm->default_wm;
 765
 766         /*
 767          * Bspec seems to indicate that the value shouldn't be lower than
 768          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 769          * Lets go for 8 which is the burst size since certain platforms
 770          * already use a hardcoded 8 (which is what the spec says should be
 771          * done).
 772          */
 773         if (wm_size <= 8)
 774                 wm_size = 8;
 775
 776         return wm_size;
 777 }
 778
 779 static bool is_disabling(int old, int new, int threshold)
 780 {
 781         return old >= threshold && new < threshold;
 782 }
 783
 784 static bool is_enabling(int old, int new, int threshold)
 785 {
 786         return old < threshold && new >= threshold;
 787 }
 788
 789 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 790 {
 791         return dev_priv->wm.max_level + 1;
 792 }
 793
 794 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 795                                    const struct intel_plane_state *plane_state)
 796 {
 797         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 798
 799         /* FIXME check the 'enable' instead */
 800         if (!crtc_state->base.active)
 801                 return false;
 802
 803         /*
 804          * Treat cursor with fb as always visible since cursor updates
 805          * can happen faster than the vrefresh rate, and the current
 806          * watermark code doesn't handle that correctly. Cursor updates
 807          * which set/clear the fb or change the cursor size are going
 808          * to get throttled by intel_legacy_cursor_update() to work
 809          * around this problem with the watermark code.
 810          */
 811         if (plane->id == PLANE_CURSOR)
 812                 return plane_state->base.fb != NULL;
 813         else
 814                 return plane_state->base.visible;
 815 }
 816
 817 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 818 {
 819         struct intel_crtc *crtc, *enabled = NULL;
 820
 821         for_each_intel_crtc(&dev_priv->drm, crtc) {
 822                 if (intel_crtc_active(crtc)) {
 823                         if (enabled)
 824                                 return NULL;
 825                         enabled = crtc;
 826                 }
 827         }
 828
 829         return enabled;
 830 }
 831
 832 static void pineview_update_wm(struct intel_crtc *unused_crtc)
 833 {
 834         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 835         struct intel_crtc *crtc;
 836         const struct cxsr_latency *latency;
 837         u32 reg;
 838         unsigned int wm;
 839
 840         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 841                                          dev_priv->is_ddr3,
 842                                          dev_priv->fsb_freq,
 843                                          dev_priv->mem_freq);
 844         if (!latency) {
 845                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 846                 intel_set_memory_cxsr(dev_priv, false);
 847                 return;
 848         }
 849
 850         crtc = single_enabled_crtc(dev_priv);
 851         if (crtc) {
 852                 const struct drm_display_mode *adjusted_mode =
 853                         &crtc->config->base.adjusted_mode;
 854                 const struct drm_framebuffer *fb =
 855                         crtc->base.primary->state->fb;
 856                 int cpp = fb->format->cpp[0];
 857                 int clock = adjusted_mode->crtc_clock;
 858
 859                 /* Display SR */
 860                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 861                                         pineview_display_wm.fifo_size,
 862                                         cpp, latency->display_sr);
 863                 reg = I915_READ(DSPFW1);
 864                 reg &= ~DSPFW_SR_MASK;
 865                 reg |= FW_WM(wm, SR);
 866                 I915_WRITE(DSPFW1, reg);
 867                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 868
 869                 /* cursor SR */
 870                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 871                                         pineview_display_wm.fifo_size,
 872                                         4, latency->cursor_sr);
 873                 reg = I915_READ(DSPFW3);
 874                 reg &= ~DSPFW_CURSOR_SR_MASK;
 875                 reg |= FW_WM(wm, CURSOR_SR);
 876                 I915_WRITE(DSPFW3, reg);
 877
 878                 /* Display HPLL off SR */
 879                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 880                                         pineview_display_hplloff_wm.fifo_size,
 881                                         cpp, latency->display_hpll_disable);
 882                 reg = I915_READ(DSPFW3);
 883                 reg &= ~DSPFW_HPLL_SR_MASK;
 884                 reg |= FW_WM(wm, HPLL_SR);
 885                 I915_WRITE(DSPFW3, reg);
 886
 887                 /* cursor HPLL off SR */
 888                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 889                                         pineview_display_hplloff_wm.fifo_size,
 890                                         4, latency->cursor_hpll_disable);
 891                 reg = I915_READ(DSPFW3);
 892                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 893                 reg |= FW_WM(wm, HPLL_CURSOR);
 894                 I915_WRITE(DSPFW3, reg);
 895                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 896
 897                 intel_set_memory_cxsr(dev_priv, true);
 898         } else {
 899                 intel_set_memory_cxsr(dev_priv, false);
 900         }
 901 }
 902
 903 /*
 904  * Documentation says:
 905  * "If the line size is small, the TLB fetches can get in the way of the
 906  *  data fetches, causing some lag in the pixel data return which is not
 907  *  accounted for in the above formulas. The following adjustment only
 908  *  needs to be applied if eight whole lines fit in the buffer at once.
 909  *  The WM is adjusted upwards by the difference between the FIFO size
 910  *  and the size of 8 whole lines. This adjustment is always performed
 911  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 912  */
 913 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 914 {
 915         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 916
 917         return max(0, tlb_miss);
 918 }
 919
 920 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 921                                 const struct g4x_wm_values *wm)
 922 {
 923         enum pipe pipe;
 924
 925         for_each_pipe(dev_priv, pipe)
 926                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 927
 928         I915_WRITE(DSPFW1,
 929                    FW_WM(wm->sr.plane, SR) |
 930                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 931                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 932                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 933         I915_WRITE(DSPFW2,
 934                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 935                    FW_WM(wm->sr.fbc, FBC_SR) |
 936                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 937                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 938                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 939                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 940         I915_WRITE(DSPFW3,
 941                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 942                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 943                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 944                    FW_WM(wm->hpll.plane, HPLL_SR));
 945
 946         POSTING_READ(DSPFW1);
 947 }
 948
 949 #define FW_WM_VLV(value, plane) \
 950         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 951
 952 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 953                                 const struct vlv_wm_values *wm)
 954 {
 955         enum pipe pipe;
 956
 957         for_each_pipe(dev_priv, pipe) {
 958                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 959
 960                 I915_WRITE(VLV_DDL(pipe),
 961                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 962                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 963                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 964                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 965         }
 966
 967         /*
 968          * Zero the (unused) WM1 watermarks, and also clear all the
 969          * high order bits so that there are no out of bounds values
 970          * present in the registers during the reprogramming.
 971          */
 972         I915_WRITE(DSPHOWM, 0);
 973         I915_WRITE(DSPHOWM1, 0);
 974         I915_WRITE(DSPFW4, 0);
 975         I915_WRITE(DSPFW5, 0);
 976         I915_WRITE(DSPFW6, 0);
 977
 978         I915_WRITE(DSPFW1,
 979                    FW_WM(wm->sr.plane, SR) |
 980                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 981                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 982                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 983         I915_WRITE(DSPFW2,
 984                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 985                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 986                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 987         I915_WRITE(DSPFW3,
 988                    FW_WM(wm->sr.cursor, CURSOR_SR));
 989
 990         if (IS_CHERRYVIEW(dev_priv)) {
 991                 I915_WRITE(DSPFW7_CHV,
 992                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 993                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 994                 I915_WRITE(DSPFW8_CHV,
 995                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
 996                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
 997                 I915_WRITE(DSPFW9_CHV,
 998                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
 999                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1000                 I915_WRITE(DSPHOWM,
1001                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1002                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1003                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1004                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1005                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1006                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1007                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1008                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1009                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1010                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1011         } else {
1012                 I915_WRITE(DSPFW7,
1013                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1014                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1015                 I915_WRITE(DSPHOWM,
1016                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1017                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1018                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1019                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1020                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1021                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1022                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1023         }
1024
1025         POSTING_READ(DSPFW1);
1026 }
1027
1028 #undef FW_WM_VLV
1029
1030 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1031 {
1032         /* all latencies in usec */
1033         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1034         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1035         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1036
1037         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1038 }
1039
1040 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1041 {
1042         /*
1043          * DSPCNTR[13] supposedly controls whether the
1044          * primary plane can use the FIFO space otherwise
1045          * reserved for the sprite plane. It's not 100% clear
1046          * what the actual FIFO size is, but it looks like we
1047          * can happily set both primary and sprite watermarks
1048          * up to 127 cachelines. So that would seem to mean
1049          * that either DSPCNTR[13] doesn't do anything, or that
1050          * the total FIFO is >= 256 cachelines in size. Either
1051          * way, we don't seem to have to worry about this
1052          * repartitioning as the maximum watermark value the
1053          * register can hold for each plane is lower than the
1054          * minimum FIFO size.
1055          */
1056         switch (plane_id) {
1057         case PLANE_CURSOR:
1058                 return 63;
1059         case PLANE_PRIMARY:
1060                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1061         case PLANE_SPRITE0:
1062                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1063         default:
1064                 MISSING_CASE(plane_id);
1065                 return 0;
1066         }
1067 }
1068
1069 static int g4x_fbc_fifo_size(int level)
1070 {
1071         switch (level) {
1072         case G4X_WM_LEVEL_SR:
1073                 return 7;
1074         case G4X_WM_LEVEL_HPLL:
1075                 return 15;
1076         default:
1077                 MISSING_CASE(level);
1078                 return 0;
1079         }
1080 }
1081
1082 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1083                           const struct intel_plane_state *plane_state,
1084                           int level)
1085 {
1086         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1087         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1088         const struct drm_display_mode *adjusted_mode =
1089                 &crtc_state->base.adjusted_mode;
1090         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1091         unsigned int clock, htotal, cpp, width, wm;
1092
1093         if (latency == 0)
1094                 return USHRT_MAX;
1095
1096         if (!intel_wm_plane_visible(crtc_state, plane_state))
1097                 return 0;
1098
1099         cpp = plane_state->base.fb->format->cpp[0];
1100
1101         /*
1102          * Not 100% sure which way ELK should go here as the
1103          * spec only says CL/CTG should assume 32bpp and BW
1104          * doesn't need to. But as these things followed the
1105          * mobile vs. desktop lines on gen3 as well, let's
1106          * assume ELK doesn't need this.
1107          *
1108          * The spec also fails to list such a restriction for
1109          * the HPLL watermark, which seems a little strange.
1110          * Let's use 32bpp for the HPLL watermark as well.
1111          */
1112         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1113             level != G4X_WM_LEVEL_NORMAL)
1114                 cpp = max(cpp, 4u);
1115
1116         clock = adjusted_mode->crtc_clock;
1117         htotal = adjusted_mode->crtc_htotal;
1118
1119         width = drm_rect_width(&plane_state->base.dst);
1120
1121         if (plane->id == PLANE_CURSOR) {
1122                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1123         } else if (plane->id == PLANE_PRIMARY &&
1124                    level == G4X_WM_LEVEL_NORMAL) {
1125                 wm = intel_wm_method1(clock, cpp, latency);
1126         } else {
1127                 unsigned int small, large;
1128
1129                 small = intel_wm_method1(clock, cpp, latency);
1130                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1131
1132                 wm = min(small, large);
1133         }
1134
1135         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1136                               width, cpp);
1137
1138         wm = DIV_ROUND_UP(wm, 64) + 2;
1139
1140         return min_t(unsigned int, wm, USHRT_MAX);
1141 }
1142
1143 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1144                                  int level, enum plane_id plane_id, u16 value)
1145 {
1146         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1147         bool dirty = false;
1148
1149         for (; level < intel_wm_num_levels(dev_priv); level++) {
1150                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1151
1152                 dirty |= raw->plane[plane_id] != value;
1153                 raw->plane[plane_id] = value;
1154         }
1155
1156         return dirty;
1157 }
1158
1159 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1160                                int level, u16 value)
1161 {
1162         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1163         bool dirty = false;
1164
1165         /* NORMAL level doesn't have an FBC watermark */
1166         level = max(level, G4X_WM_LEVEL_SR);
1167
1168         for (; level < intel_wm_num_levels(dev_priv); level++) {
1169                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1170
1171                 dirty |= raw->fbc != value;
1172                 raw->fbc = value;
1173         }
1174
1175         return dirty;
1176 }
1177
1178 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1179                               const struct intel_plane_state *plane_state,
1180                               u32 pri_val);
1181
1182 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1183                                      const struct intel_plane_state *plane_state)
1184 {
1185         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1186         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1187         enum plane_id plane_id = plane->id;
1188         bool dirty = false;
1189         int level;
1190
1191         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1192                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1193                 if (plane_id == PLANE_PRIMARY)
1194                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1195                 goto out;
1196         }
1197
1198         for (level = 0; level < num_levels; level++) {
1199                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1200                 int wm, max_wm;
1201
1202                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1203                 max_wm = g4x_plane_fifo_size(plane_id, level);
1204
1205                 if (wm > max_wm)
1206                         break;
1207
1208                 dirty |= raw->plane[plane_id] != wm;
1209                 raw->plane[plane_id] = wm;
1210
1211                 if (plane_id != PLANE_PRIMARY ||
1212                     level == G4X_WM_LEVEL_NORMAL)
1213                         continue;
1214
1215                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1216                                         raw->plane[plane_id]);
1217                 max_wm = g4x_fbc_fifo_size(level);
1218
1219                 /*
1220                  * FBC wm is not mandatory as we
1221                  * can always just disable its use.
1222                  */
1223                 if (wm > max_wm)
1224                         wm = USHRT_MAX;
1225
1226                 dirty |= raw->fbc != wm;
1227                 raw->fbc = wm;
1228         }
1229
1230         /* mark watermarks as invalid */
1231         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1232
1233         if (plane_id == PLANE_PRIMARY)
1234                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1235
1236  out:
1237         if (dirty) {
1238                 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1239                               plane->base.name,
1240                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1241                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1242                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1243
1244                 if (plane_id == PLANE_PRIMARY)
1245                         DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1246                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1247                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1248         }
1249
1250         return dirty;
1251 }
1252
1253 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1254                                       enum plane_id plane_id, int level)
1255 {
1256         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1257
1258         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1259 }
1260
1261 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1262                                      int level)
1263 {
1264         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1265
1266         if (level > dev_priv->wm.max_level)
1267                 return false;
1268
1269         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1270                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1271                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1272 }
1273
1274 /* mark all levels starting from 'level' as invalid */
1275 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1276                                struct g4x_wm_state *wm_state, int level)
1277 {
1278         if (level <= G4X_WM_LEVEL_NORMAL) {
1279                 enum plane_id plane_id;
1280
1281                 for_each_plane_id_on_crtc(crtc, plane_id)
1282                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1283         }
1284
1285         if (level <= G4X_WM_LEVEL_SR) {
1286                 wm_state->cxsr = false;
1287                 wm_state->sr.cursor = USHRT_MAX;
1288                 wm_state->sr.plane = USHRT_MAX;
1289                 wm_state->sr.fbc = USHRT_MAX;
1290         }
1291
1292         if (level <= G4X_WM_LEVEL_HPLL) {
1293                 wm_state->hpll_en = false;
1294                 wm_state->hpll.cursor = USHRT_MAX;
1295                 wm_state->hpll.plane = USHRT_MAX;
1296                 wm_state->hpll.fbc = USHRT_MAX;
1297         }
1298 }
1299
1300 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1301 {
1302         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1303         struct intel_atomic_state *state =
1304                 to_intel_atomic_state(crtc_state->base.state);
1305         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1306         int num_active_planes = hweight8(crtc_state->active_planes &
1307                                          ~BIT(PLANE_CURSOR));
1308         const struct g4x_pipe_wm *raw;
1309         const struct intel_plane_state *old_plane_state;
1310         const struct intel_plane_state *new_plane_state;
1311         struct intel_plane *plane;
1312         enum plane_id plane_id;
1313         int i, level;
1314         unsigned int dirty = 0;
1315
1316         for_each_oldnew_intel_plane_in_state(state, plane,
1317                                              old_plane_state,
1318                                              new_plane_state, i) {
1319                 if (new_plane_state->base.crtc != &crtc->base &&
1320                     old_plane_state->base.crtc != &crtc->base)
1321                         continue;
1322
1323                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1324                         dirty |= BIT(plane->id);
1325         }
1326
1327         if (!dirty)
1328                 return 0;
1329
1330         level = G4X_WM_LEVEL_NORMAL;
1331         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1332                 goto out;
1333
1334         raw = &crtc_state->wm.g4x.raw[level];
1335         for_each_plane_id_on_crtc(crtc, plane_id)
1336                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1337
1338         level = G4X_WM_LEVEL_SR;
1339
1340         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1341                 goto out;
1342
1343         raw = &crtc_state->wm.g4x.raw[level];
1344         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1345         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1346         wm_state->sr.fbc = raw->fbc;
1347
1348         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1349
1350         level = G4X_WM_LEVEL_HPLL;
1351
1352         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1353                 goto out;
1354
1355         raw = &crtc_state->wm.g4x.raw[level];
1356         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1357         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1358         wm_state->hpll.fbc = raw->fbc;
1359
1360         wm_state->hpll_en = wm_state->cxsr;
1361
1362         level++;
1363
1364  out:
1365         if (level == G4X_WM_LEVEL_NORMAL)
1366                 return -EINVAL;
1367
1368         /* invalidate the higher levels */
1369         g4x_invalidate_wms(crtc, wm_state, level);
1370
1371         /*
1372          * Determine if the FBC watermark(s) can be used. IF
1373          * this isn't the case we prefer to disable the FBC
1374          ( watermark(s) rather than disable the SR/HPLL
1375          * level(s) entirely.
1376          */
1377         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1378
1379         if (level >= G4X_WM_LEVEL_SR &&
1380             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1381                 wm_state->fbc_en = false;
1382         else if (level >= G4X_WM_LEVEL_HPLL &&
1383                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1384                 wm_state->fbc_en = false;
1385
1386         return 0;
1387 }
1388
1389 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1390 {
1391         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1392         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1393         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1394         struct intel_atomic_state *intel_state =
1395                 to_intel_atomic_state(new_crtc_state->base.state);
1396         const struct intel_crtc_state *old_crtc_state =
1397                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1398         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1399         enum plane_id plane_id;
1400
1401         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1402                 *intermediate = *optimal;
1403
1404                 intermediate->cxsr = false;
1405                 intermediate->hpll_en = false;
1406                 goto out;
1407         }
1408
1409         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1410                 !new_crtc_state->disable_cxsr;
1411         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1412                 !new_crtc_state->disable_cxsr;
1413         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1414
1415         for_each_plane_id_on_crtc(crtc, plane_id) {
1416                 intermediate->wm.plane[plane_id] =
1417                         max(optimal->wm.plane[plane_id],
1418                             active->wm.plane[plane_id]);
1419
1420                 WARN_ON(intermediate->wm.plane[plane_id] >
1421                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1422         }
1423
1424         intermediate->sr.plane = max(optimal->sr.plane,
1425                                      active->sr.plane);
1426         intermediate->sr.cursor = max(optimal->sr.cursor,
1427                                       active->sr.cursor);
1428         intermediate->sr.fbc = max(optimal->sr.fbc,
1429                                    active->sr.fbc);
1430
1431         intermediate->hpll.plane = max(optimal->hpll.plane,
1432                                        active->hpll.plane);
1433         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1434                                         active->hpll.cursor);
1435         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1436                                      active->hpll.fbc);
1437
1438         WARN_ON((intermediate->sr.plane >
1439                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1440                  intermediate->sr.cursor >
1441                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1442                 intermediate->cxsr);
1443         WARN_ON((intermediate->sr.plane >
1444                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1445                  intermediate->sr.cursor >
1446                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1447                 intermediate->hpll_en);
1448
1449         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1450                 intermediate->fbc_en && intermediate->cxsr);
1451         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1452                 intermediate->fbc_en && intermediate->hpll_en);
1453
1454 out:
1455         /*
1456          * If our intermediate WM are identical to the final WM, then we can
1457          * omit the post-vblank programming; only update if it's different.
1458          */
1459         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1460                 new_crtc_state->wm.need_postvbl_update = true;
1461
1462         return 0;
1463 }
1464
1465 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1466                          struct g4x_wm_values *wm)
1467 {
1468         struct intel_crtc *crtc;
1469         int num_active_pipes = 0;
1470
1471         wm->cxsr = true;
1472         wm->hpll_en = true;
1473         wm->fbc_en = true;
1474
1475         for_each_intel_crtc(&dev_priv->drm, crtc) {
1476                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1477
1478                 if (!crtc->active)
1479                         continue;
1480
1481                 if (!wm_state->cxsr)
1482                         wm->cxsr = false;
1483                 if (!wm_state->hpll_en)
1484                         wm->hpll_en = false;
1485                 if (!wm_state->fbc_en)
1486                         wm->fbc_en = false;
1487
1488                 num_active_pipes++;
1489         }
1490
1491         if (num_active_pipes != 1) {
1492                 wm->cxsr = false;
1493                 wm->hpll_en = false;
1494                 wm->fbc_en = false;
1495         }
1496
1497         for_each_intel_crtc(&dev_priv->drm, crtc) {
1498                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1499                 enum pipe pipe = crtc->pipe;
1500
1501                 wm->pipe[pipe] = wm_state->wm;
1502                 if (crtc->active && wm->cxsr)
1503                         wm->sr = wm_state->sr;
1504                 if (crtc->active && wm->hpll_en)
1505                         wm->hpll = wm_state->hpll;
1506         }
1507 }
1508
1509 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1510 {
1511         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1512         struct g4x_wm_values new_wm = {};
1513
1514         g4x_merge_wm(dev_priv, &new_wm);
1515
1516         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1517                 return;
1518
1519         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1520                 _intel_set_memory_cxsr(dev_priv, false);
1521
1522         g4x_write_wm_values(dev_priv, &new_wm);
1523
1524         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1525                 _intel_set_memory_cxsr(dev_priv, true);
1526
1527         *old_wm = new_wm;
1528 }
1529
1530 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1531                                    struct intel_crtc_state *crtc_state)
1532 {
1533         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1534         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1535
1536         mutex_lock(&dev_priv->wm.wm_mutex);
1537         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1538         g4x_program_watermarks(dev_priv);
1539         mutex_unlock(&dev_priv->wm.wm_mutex);
1540 }
1541
1542 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1543                                     struct intel_crtc_state *crtc_state)
1544 {
1545         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1546         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1547
1548         if (!crtc_state->wm.need_postvbl_update)
1549                 return;
1550
1551         mutex_lock(&dev_priv->wm.wm_mutex);
1552         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1553         g4x_program_watermarks(dev_priv);
1554         mutex_unlock(&dev_priv->wm.wm_mutex);
1555 }
1556
1557 /* latency must be in 0.1us units. */
1558 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1559                                    unsigned int htotal,
1560                                    unsigned int width,
1561                                    unsigned int cpp,
1562                                    unsigned int latency)
1563 {
1564         unsigned int ret;
1565
1566         ret = intel_wm_method2(pixel_rate, htotal,
1567                                width, cpp, latency);
1568         ret = DIV_ROUND_UP(ret, 64);
1569
1570         return ret;
1571 }
1572
1573 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1574 {
1575         /* all latencies in usec */
1576         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1577
1578         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1579
1580         if (IS_CHERRYVIEW(dev_priv)) {
1581                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1582                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1583
1584                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1585         }
1586 }
1587
1588 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1589                                 const struct intel_plane_state *plane_state,
1590                                 int level)
1591 {
1592         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1593         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1594         const struct drm_display_mode *adjusted_mode =
1595                 &crtc_state->base.adjusted_mode;
1596         unsigned int clock, htotal, cpp, width, wm;
1597
1598         if (dev_priv->wm.pri_latency[level] == 0)
1599                 return USHRT_MAX;
1600
1601         if (!intel_wm_plane_visible(crtc_state, plane_state))
1602                 return 0;
1603
1604         cpp = plane_state->base.fb->format->cpp[0];
1605         clock = adjusted_mode->crtc_clock;
1606         htotal = adjusted_mode->crtc_htotal;
1607         width = crtc_state->pipe_src_w;
1608
1609         if (plane->id == PLANE_CURSOR) {
1610                 /*
1611                  * FIXME the formula gives values that are
1612                  * too big for the cursor FIFO, and hence we
1613                  * would never be able to use cursors. For
1614                  * now just hardcode the watermark.
1615                  */
1616                 wm = 63;
1617         } else {
1618                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1619                                     dev_priv->wm.pri_latency[level] * 10);
1620         }
1621
1622         return min_t(unsigned int, wm, USHRT_MAX);
1623 }
1624
1625 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1626 {
1627         return (active_planes & (BIT(PLANE_SPRITE0) |
1628                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1629 }
1630
1631 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1632 {
1633         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1634         const struct g4x_pipe_wm *raw =
1635                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1636         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1637         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1638         int num_active_planes = hweight8(active_planes);
1639         const int fifo_size = 511;
1640         int fifo_extra, fifo_left = fifo_size;
1641         int sprite0_fifo_extra = 0;
1642         unsigned int total_rate;
1643         enum plane_id plane_id;
1644
1645         /*
1646          * When enabling sprite0 after sprite1 has already been enabled
1647          * we tend to get an underrun unless sprite0 already has some
1648          * FIFO space allcoated. Hence we always allocate at least one
1649          * cacheline for sprite0 whenever sprite1 is enabled.
1650          *
1651          * All other plane enable sequences appear immune to this problem.
1652          */
1653         if (vlv_need_sprite0_fifo_workaround(active_planes))
1654                 sprite0_fifo_extra = 1;
1655
1656         total_rate = raw->plane[PLANE_PRIMARY] +
1657                 raw->plane[PLANE_SPRITE0] +
1658                 raw->plane[PLANE_SPRITE1] +
1659                 sprite0_fifo_extra;
1660
1661         if (total_rate > fifo_size)
1662                 return -EINVAL;
1663
1664         if (total_rate == 0)
1665                 total_rate = 1;
1666
1667         for_each_plane_id_on_crtc(crtc, plane_id) {
1668                 unsigned int rate;
1669
1670                 if ((active_planes & BIT(plane_id)) == 0) {
1671                         fifo_state->plane[plane_id] = 0;
1672                         continue;
1673                 }
1674
1675                 rate = raw->plane[plane_id];
1676                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1677                 fifo_left -= fifo_state->plane[plane_id];
1678         }
1679
1680         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1681         fifo_left -= sprite0_fifo_extra;
1682
1683         fifo_state->plane[PLANE_CURSOR] = 63;
1684
1685         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1686
1687         /* spread the remainder evenly */
1688         for_each_plane_id_on_crtc(crtc, plane_id) {
1689                 int plane_extra;
1690
1691                 if (fifo_left == 0)
1692                         break;
1693
1694                 if ((active_planes & BIT(plane_id)) == 0)
1695                         continue;
1696
1697                 plane_extra = min(fifo_extra, fifo_left);
1698                 fifo_state->plane[plane_id] += plane_extra;
1699                 fifo_left -= plane_extra;
1700         }
1701
1702         WARN_ON(active_planes != 0 && fifo_left != 0);
1703
1704         /* give it all to the first plane if none are active */
1705         if (active_planes == 0) {
1706                 WARN_ON(fifo_left != fifo_size);
1707                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1708         }
1709
1710         return 0;
1711 }
1712
1713 /* mark all levels starting from 'level' as invalid */
1714 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1715                                struct vlv_wm_state *wm_state, int level)
1716 {
1717         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1718
1719         for (; level < intel_wm_num_levels(dev_priv); level++) {
1720                 enum plane_id plane_id;
1721
1722                 for_each_plane_id_on_crtc(crtc, plane_id)
1723                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1724
1725                 wm_state->sr[level].cursor = USHRT_MAX;
1726                 wm_state->sr[level].plane = USHRT_MAX;
1727         }
1728 }
1729
1730 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1731 {
1732         if (wm > fifo_size)
1733                 return USHRT_MAX;
1734         else
1735                 return fifo_size - wm;
1736 }
1737
1738 /*
1739  * Starting from 'level' set all higher
1740  * levels to 'value' in the "raw" watermarks.
1741  */
1742 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1743                                  int level, enum plane_id plane_id, u16 value)
1744 {
1745         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1746         int num_levels = intel_wm_num_levels(dev_priv);
1747         bool dirty = false;
1748
1749         for (; level < num_levels; level++) {
1750                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1751
1752                 dirty |= raw->plane[plane_id] != value;
1753                 raw->plane[plane_id] = value;
1754         }
1755
1756         return dirty;
1757 }
1758
1759 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1760                                      const struct intel_plane_state *plane_state)
1761 {
1762         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1763         enum plane_id plane_id = plane->id;
1764         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1765         int level;
1766         bool dirty = false;
1767
1768         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1769                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1770                 goto out;
1771         }
1772
1773         for (level = 0; level < num_levels; level++) {
1774                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1775                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1776                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1777
1778                 if (wm > max_wm)
1779                         break;
1780
1781                 dirty |= raw->plane[plane_id] != wm;
1782                 raw->plane[plane_id] = wm;
1783         }
1784
1785         /* mark all higher levels as invalid */
1786         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1787
1788 out:
1789         if (dirty)
1790                 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1791                               plane->base.name,
1792                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1793                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1794                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1795
1796         return dirty;
1797 }
1798
1799 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1800                                       enum plane_id plane_id, int level)
1801 {
1802         const struct g4x_pipe_wm *raw =
1803                 &crtc_state->wm.vlv.raw[level];
1804         const struct vlv_fifo_state *fifo_state =
1805                 &crtc_state->wm.vlv.fifo_state;
1806
1807         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1808 }
1809
1810 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1811 {
1812         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1813                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1814                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1815                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1816 }
1817
1818 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1819 {
1820         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1821         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1822         struct intel_atomic_state *state =
1823                 to_intel_atomic_state(crtc_state->base.state);
1824         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1825         const struct vlv_fifo_state *fifo_state =
1826                 &crtc_state->wm.vlv.fifo_state;
1827         int num_active_planes = hweight8(crtc_state->active_planes &
1828                                          ~BIT(PLANE_CURSOR));
1829         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1830         const struct intel_plane_state *old_plane_state;
1831         const struct intel_plane_state *new_plane_state;
1832         struct intel_plane *plane;
1833         enum plane_id plane_id;
1834         int level, ret, i;
1835         unsigned int dirty = 0;
1836
1837         for_each_oldnew_intel_plane_in_state(state, plane,
1838                                              old_plane_state,
1839                                              new_plane_state, i) {
1840                 if (new_plane_state->base.crtc != &crtc->base &&
1841                     old_plane_state->base.crtc != &crtc->base)
1842                         continue;
1843
1844                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1845                         dirty |= BIT(plane->id);
1846         }
1847
1848         /*
1849          * DSPARB registers may have been reset due to the
1850          * power well being turned off. Make sure we restore
1851          * them to a consistent state even if no primary/sprite
1852          * planes are initially active.
1853          */
1854         if (needs_modeset)
1855                 crtc_state->fifo_changed = true;
1856
1857         if (!dirty)
1858                 return 0;
1859
1860         /* cursor changes don't warrant a FIFO recompute */
1861         if (dirty & ~BIT(PLANE_CURSOR)) {
1862                 const struct intel_crtc_state *old_crtc_state =
1863                         intel_atomic_get_old_crtc_state(state, crtc);
1864                 const struct vlv_fifo_state *old_fifo_state =
1865                         &old_crtc_state->wm.vlv.fifo_state;
1866
1867                 ret = vlv_compute_fifo(crtc_state);
1868                 if (ret)
1869                         return ret;
1870
1871                 if (needs_modeset ||
1872                     memcmp(old_fifo_state, fifo_state,
1873                            sizeof(*fifo_state)) != 0)
1874                         crtc_state->fifo_changed = true;
1875         }
1876
1877         /* initially allow all levels */
1878         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1879         /*
1880          * Note that enabling cxsr with no primary/sprite planes
1881          * enabled can wedge the pipe. Hence we only allow cxsr
1882          * with exactly one enabled primary/sprite plane.
1883          */
1884         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1885
1886         for (level = 0; level < wm_state->num_levels; level++) {
1887                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1888                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1889
1890                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1891                         break;
1892
1893                 for_each_plane_id_on_crtc(crtc, plane_id) {
1894                         wm_state->wm[level].plane[plane_id] =
1895                                 vlv_invert_wm_value(raw->plane[plane_id],
1896                                                     fifo_state->plane[plane_id]);
1897                 }
1898
1899                 wm_state->sr[level].plane =
1900                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1901                                                  raw->plane[PLANE_SPRITE0],
1902                                                  raw->plane[PLANE_SPRITE1]),
1903                                             sr_fifo_size);
1904
1905                 wm_state->sr[level].cursor =
1906                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1907                                             63);
1908         }
1909
1910         if (level == 0)
1911                 return -EINVAL;
1912
1913         /* limit to only levels we can actually handle */
1914         wm_state->num_levels = level;
1915
1916         /* invalidate the higher levels */
1917         vlv_invalidate_wms(crtc, wm_state, level);
1918
1919         return 0;
1920 }
1921
1922 #define VLV_FIFO(plane, value) \
1923         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1924
1925 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1926                                    struct intel_crtc_state *crtc_state)
1927 {
1928         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1929         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1930         struct intel_uncore *uncore = &dev_priv->uncore;
1931         const struct vlv_fifo_state *fifo_state =
1932                 &crtc_state->wm.vlv.fifo_state;
1933         int sprite0_start, sprite1_start, fifo_size;
1934
1935         if (!crtc_state->fifo_changed)
1936                 return;
1937
1938         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1939         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1940         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1941
1942         WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1943         WARN_ON(fifo_size != 511);
1944
1945         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1946
1947         /*
1948          * uncore.lock serves a double purpose here. It allows us to
1949          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1950          * it protects the DSPARB registers from getting clobbered by
1951          * parallel updates from multiple pipes.
1952          *
1953          * intel_pipe_update_start() has already disabled interrupts
1954          * for us, so a plain spin_lock() is sufficient here.
1955          */
1956         spin_lock(&uncore->lock);
1957
1958         switch (crtc->pipe) {
1959                 u32 dsparb, dsparb2, dsparb3;
1960         case PIPE_A:
1961                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1962                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1963
1964                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1965                             VLV_FIFO(SPRITEB, 0xff));
1966                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1967                            VLV_FIFO(SPRITEB, sprite1_start));
1968
1969                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1970                              VLV_FIFO(SPRITEB_HI, 0x1));
1971                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1972                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1973
1974                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1975                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1976                 break;
1977         case PIPE_B:
1978                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1979                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1980
1981                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1982                             VLV_FIFO(SPRITED, 0xff));
1983                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1984                            VLV_FIFO(SPRITED, sprite1_start));
1985
1986                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1987                              VLV_FIFO(SPRITED_HI, 0xff));
1988                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1989                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1990
1991                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1992                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1993                 break;
1994         case PIPE_C:
1995                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1996                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1997
1998                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1999                              VLV_FIFO(SPRITEF, 0xff));
2000                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2001                             VLV_FIFO(SPRITEF, sprite1_start));
2002
2003                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2004                              VLV_FIFO(SPRITEF_HI, 0xff));
2005                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2006                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2007
2008                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2009                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2010                 break;
2011         default:
2012                 break;
2013         }
2014
2015         intel_uncore_posting_read_fw(uncore, DSPARB);
2016
2017         spin_unlock(&uncore->lock);
2018 }
2019
2020 #undef VLV_FIFO
2021
2022 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2023 {
2024         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
2025         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2026         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2027         struct intel_atomic_state *intel_state =
2028                 to_intel_atomic_state(new_crtc_state->base.state);
2029         const struct intel_crtc_state *old_crtc_state =
2030                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2031         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2032         int level;
2033
2034         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2035                 *intermediate = *optimal;
2036
2037                 intermediate->cxsr = false;
2038                 goto out;
2039         }
2040
2041         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2042         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2043                 !new_crtc_state->disable_cxsr;
2044
2045         for (level = 0; level < intermediate->num_levels; level++) {
2046                 enum plane_id plane_id;
2047
2048                 for_each_plane_id_on_crtc(crtc, plane_id) {
2049                         intermediate->wm[level].plane[plane_id] =
2050                                 min(optimal->wm[level].plane[plane_id],
2051                                     active->wm[level].plane[plane_id]);
2052                 }
2053
2054                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2055                                                     active->sr[level].plane);
2056                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2057                                                      active->sr[level].cursor);
2058         }
2059
2060         vlv_invalidate_wms(crtc, intermediate, level);
2061
2062 out:
2063         /*
2064          * If our intermediate WM are identical to the final WM, then we can
2065          * omit the post-vblank programming; only update if it's different.
2066          */
2067         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2068                 new_crtc_state->wm.need_postvbl_update = true;
2069
2070         return 0;
2071 }
2072
2073 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2074                          struct vlv_wm_values *wm)
2075 {
2076         struct intel_crtc *crtc;
2077         int num_active_pipes = 0;
2078
2079         wm->level = dev_priv->wm.max_level;
2080         wm->cxsr = true;
2081
2082         for_each_intel_crtc(&dev_priv->drm, crtc) {
2083                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2084
2085                 if (!crtc->active)
2086                         continue;
2087
2088                 if (!wm_state->cxsr)
2089                         wm->cxsr = false;
2090
2091                 num_active_pipes++;
2092                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2093         }
2094
2095         if (num_active_pipes != 1)
2096                 wm->cxsr = false;
2097
2098         if (num_active_pipes > 1)
2099                 wm->level = VLV_WM_LEVEL_PM2;
2100
2101         for_each_intel_crtc(&dev_priv->drm, crtc) {
2102                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2103                 enum pipe pipe = crtc->pipe;
2104
2105                 wm->pipe[pipe] = wm_state->wm[wm->level];
2106                 if (crtc->active && wm->cxsr)
2107                         wm->sr = wm_state->sr[wm->level];
2108
2109                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2110                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2111                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2112                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2113         }
2114 }
2115
2116 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2117 {
2118         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2119         struct vlv_wm_values new_wm = {};
2120
2121         vlv_merge_wm(dev_priv, &new_wm);
2122
2123         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2124                 return;
2125
2126         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2127                 chv_set_memory_dvfs(dev_priv, false);
2128
2129         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2130                 chv_set_memory_pm5(dev_priv, false);
2131
2132         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2133                 _intel_set_memory_cxsr(dev_priv, false);
2134
2135         vlv_write_wm_values(dev_priv, &new_wm);
2136
2137         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2138                 _intel_set_memory_cxsr(dev_priv, true);
2139
2140         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2141                 chv_set_memory_pm5(dev_priv, true);
2142
2143         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2144                 chv_set_memory_dvfs(dev_priv, true);
2145
2146         *old_wm = new_wm;
2147 }
2148
2149 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2150                                    struct intel_crtc_state *crtc_state)
2151 {
2152         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2153         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2154
2155         mutex_lock(&dev_priv->wm.wm_mutex);
2156         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2157         vlv_program_watermarks(dev_priv);
2158         mutex_unlock(&dev_priv->wm.wm_mutex);
2159 }
2160
2161 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2162                                     struct intel_crtc_state *crtc_state)
2163 {
2164         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2165         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2166
2167         if (!crtc_state->wm.need_postvbl_update)
2168                 return;
2169
2170         mutex_lock(&dev_priv->wm.wm_mutex);
2171         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2172         vlv_program_watermarks(dev_priv);
2173         mutex_unlock(&dev_priv->wm.wm_mutex);
2174 }
2175
2176 static void i965_update_wm(struct intel_crtc *unused_crtc)
2177 {
2178         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2179         struct intel_crtc *crtc;
2180         int srwm = 1;
2181         int cursor_sr = 16;
2182         bool cxsr_enabled;
2183
2184         /* Calc sr entries for one plane configs */
2185         crtc = single_enabled_crtc(dev_priv);
2186         if (crtc) {
2187                 /* self-refresh has much higher latency */
2188                 static const int sr_latency_ns = 12000;
2189                 const struct drm_display_mode *adjusted_mode =
2190                         &crtc->config->base.adjusted_mode;
2191                 const struct drm_framebuffer *fb =
2192                         crtc->base.primary->state->fb;
2193                 int clock = adjusted_mode->crtc_clock;
2194                 int htotal = adjusted_mode->crtc_htotal;
2195                 int hdisplay = crtc->config->pipe_src_w;
2196                 int cpp = fb->format->cpp[0];
2197                 int entries;
2198
2199                 entries = intel_wm_method2(clock, htotal,
2200                                            hdisplay, cpp, sr_latency_ns / 100);
2201                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2202                 srwm = I965_FIFO_SIZE - entries;
2203                 if (srwm < 0)
2204                         srwm = 1;
2205                 srwm &= 0x1ff;
2206                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2207                               entries, srwm);
2208
2209                 entries = intel_wm_method2(clock, htotal,
2210                                            crtc->base.cursor->state->crtc_w, 4,
2211                                            sr_latency_ns / 100);
2212                 entries = DIV_ROUND_UP(entries,
2213                                        i965_cursor_wm_info.cacheline_size) +
2214                         i965_cursor_wm_info.guard_size;
2215
2216                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2217                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2218                         cursor_sr = i965_cursor_wm_info.max_wm;
2219
2220                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2221                               "cursor %d\n", srwm, cursor_sr);
2222
2223                 cxsr_enabled = true;
2224         } else {
2225                 cxsr_enabled = false;
2226                 /* Turn off self refresh if both pipes are enabled */
2227                 intel_set_memory_cxsr(dev_priv, false);
2228         }
2229
2230         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2231                       srwm);
2232
2233         /* 965 has limitations... */
2234         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2235                    FW_WM(8, CURSORB) |
2236                    FW_WM(8, PLANEB) |
2237                    FW_WM(8, PLANEA));
2238         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2239                    FW_WM(8, PLANEC_OLD));
2240         /* update cursor SR watermark */
2241         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2242
2243         if (cxsr_enabled)
2244                 intel_set_memory_cxsr(dev_priv, true);
2245 }
2246
2247 #undef FW_WM
2248
2249 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2250 {
2251         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2252         const struct intel_watermark_params *wm_info;
2253         u32 fwater_lo;
2254         u32 fwater_hi;
2255         int cwm, srwm = 1;
2256         int fifo_size;
2257         int planea_wm, planeb_wm;
2258         struct intel_crtc *crtc, *enabled = NULL;
2259
2260         if (IS_I945GM(dev_priv))
2261                 wm_info = &i945_wm_info;
2262         else if (!IS_GEN(dev_priv, 2))
2263                 wm_info = &i915_wm_info;
2264         else
2265                 wm_info = &i830_a_wm_info;
2266
2267         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2268         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2269         if (intel_crtc_active(crtc)) {
2270                 const struct drm_display_mode *adjusted_mode =
2271                         &crtc->config->base.adjusted_mode;
2272                 const struct drm_framebuffer *fb =
2273                         crtc->base.primary->state->fb;
2274                 int cpp;
2275
2276                 if (IS_GEN(dev_priv, 2))
2277                         cpp = 4;
2278                 else
2279                         cpp = fb->format->cpp[0];
2280
2281                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2282                                                wm_info, fifo_size, cpp,
2283                                                pessimal_latency_ns);
2284                 enabled = crtc;
2285         } else {
2286                 planea_wm = fifo_size - wm_info->guard_size;
2287                 if (planea_wm > (long)wm_info->max_wm)
2288                         planea_wm = wm_info->max_wm;
2289         }
2290
2291         if (IS_GEN(dev_priv, 2))
2292                 wm_info = &i830_bc_wm_info;
2293
2294         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2295         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2296         if (intel_crtc_active(crtc)) {
2297                 const struct drm_display_mode *adjusted_mode =
2298                         &crtc->config->base.adjusted_mode;
2299                 const struct drm_framebuffer *fb =
2300                         crtc->base.primary->state->fb;
2301                 int cpp;
2302
2303                 if (IS_GEN(dev_priv, 2))
2304                         cpp = 4;
2305                 else
2306                         cpp = fb->format->cpp[0];
2307
2308                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2309                                                wm_info, fifo_size, cpp,
2310                                                pessimal_latency_ns);
2311                 if (enabled == NULL)
2312                         enabled = crtc;
2313                 else
2314                         enabled = NULL;
2315         } else {
2316                 planeb_wm = fifo_size - wm_info->guard_size;
2317                 if (planeb_wm > (long)wm_info->max_wm)
2318                         planeb_wm = wm_info->max_wm;
2319         }
2320
2321         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2322
2323         if (IS_I915GM(dev_priv) && enabled) {
2324                 struct drm_i915_gem_object *obj;
2325
2326                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2327
2328                 /* self-refresh seems busted with untiled */
2329                 if (!i915_gem_object_is_tiled(obj))
2330                         enabled = NULL;
2331         }
2332
2333         /*
2334          * Overlay gets an aggressive default since video jitter is bad.
2335          */
2336         cwm = 2;
2337
2338         /* Play safe and disable self-refresh before adjusting watermarks. */
2339         intel_set_memory_cxsr(dev_priv, false);
2340
2341         /* Calc sr entries for one plane configs */
2342         if (HAS_FW_BLC(dev_priv) && enabled) {
2343                 /* self-refresh has much higher latency */
2344                 static const int sr_latency_ns = 6000;
2345                 const struct drm_display_mode *adjusted_mode =
2346                         &enabled->config->base.adjusted_mode;
2347                 const struct drm_framebuffer *fb =
2348                         enabled->base.primary->state->fb;
2349                 int clock = adjusted_mode->crtc_clock;
2350                 int htotal = adjusted_mode->crtc_htotal;
2351                 int hdisplay = enabled->config->pipe_src_w;
2352                 int cpp;
2353                 int entries;
2354
2355                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2356                         cpp = 4;
2357                 else
2358                         cpp = fb->format->cpp[0];
2359
2360                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2361                                            sr_latency_ns / 100);
2362                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2363                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2364                 srwm = wm_info->fifo_size - entries;
2365                 if (srwm < 0)
2366                         srwm = 1;
2367
2368                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2369                         I915_WRITE(FW_BLC_SELF,
2370                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2371                 else
2372                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2373         }
2374
2375         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2376                       planea_wm, planeb_wm, cwm, srwm);
2377
2378         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2379         fwater_hi = (cwm & 0x1f);
2380
2381         /* Set request length to 8 cachelines per fetch */
2382         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2383         fwater_hi = fwater_hi | (1 << 8);
2384
2385         I915_WRITE(FW_BLC, fwater_lo);
2386         I915_WRITE(FW_BLC2, fwater_hi);
2387
2388         if (enabled)
2389                 intel_set_memory_cxsr(dev_priv, true);
2390 }
2391
2392 static void i845_update_wm(struct intel_crtc *unused_crtc)
2393 {
2394         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2395         struct intel_crtc *crtc;
2396         const struct drm_display_mode *adjusted_mode;
2397         u32 fwater_lo;
2398         int planea_wm;
2399
2400         crtc = single_enabled_crtc(dev_priv);
2401         if (crtc == NULL)
2402                 return;
2403
2404         adjusted_mode = &crtc->config->base.adjusted_mode;
2405         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2406                                        &i845_wm_info,
2407                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2408                                        4, pessimal_latency_ns);
2409         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2410         fwater_lo |= (3<<8) | planea_wm;
2411
2412         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2413
2414         I915_WRITE(FW_BLC, fwater_lo);
2415 }
2416
2417 /* latency must be in 0.1us units. */
2418 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2419                                    unsigned int cpp,
2420                                    unsigned int latency)
2421 {
2422         unsigned int ret;
2423
2424         ret = intel_wm_method1(pixel_rate, cpp, latency);
2425         ret = DIV_ROUND_UP(ret, 64) + 2;
2426
2427         return ret;
2428 }
2429
2430 /* latency must be in 0.1us units. */
2431 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2432                                    unsigned int htotal,
2433                                    unsigned int width,
2434                                    unsigned int cpp,
2435                                    unsigned int latency)
2436 {
2437         unsigned int ret;
2438
2439         ret = intel_wm_method2(pixel_rate, htotal,
2440                                width, cpp, latency);
2441         ret = DIV_ROUND_UP(ret, 64) + 2;
2442
2443         return ret;
2444 }
2445
2446 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2447 {
2448         /*
2449          * Neither of these should be possible since this function shouldn't be
2450          * called if the CRTC is off or the plane is invisible.  But let's be
2451          * extra paranoid to avoid a potential divide-by-zero if we screw up
2452          * elsewhere in the driver.
2453          */
2454         if (WARN_ON(!cpp))
2455                 return 0;
2456         if (WARN_ON(!horiz_pixels))
2457                 return 0;
2458
2459         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2460 }
2461
2462 struct ilk_wm_maximums {
2463         u16 pri;
2464         u16 spr;
2465         u16 cur;
2466         u16 fbc;
2467 };
2468
2469 /*
2470  * For both WM_PIPE and WM_LP.
2471  * mem_value must be in 0.1us units.
2472  */
2473 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2474                               const struct intel_plane_state *plane_state,
2475                               u32 mem_value, bool is_lp)
2476 {
2477         u32 method1, method2;
2478         int cpp;
2479
2480         if (mem_value == 0)
2481                 return U32_MAX;
2482
2483         if (!intel_wm_plane_visible(crtc_state, plane_state))
2484                 return 0;
2485
2486         cpp = plane_state->base.fb->format->cpp[0];
2487
2488         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2489
2490         if (!is_lp)
2491                 return method1;
2492
2493         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2494                                  crtc_state->base.adjusted_mode.crtc_htotal,
2495                                  drm_rect_width(&plane_state->base.dst),
2496                                  cpp, mem_value);
2497
2498         return min(method1, method2);
2499 }
2500
2501 /*
2502  * For both WM_PIPE and WM_LP.
2503  * mem_value must be in 0.1us units.
2504  */
2505 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2506                               const struct intel_plane_state *plane_state,
2507                               u32 mem_value)
2508 {
2509         u32 method1, method2;
2510         int cpp;
2511
2512         if (mem_value == 0)
2513                 return U32_MAX;
2514
2515         if (!intel_wm_plane_visible(crtc_state, plane_state))
2516                 return 0;
2517
2518         cpp = plane_state->base.fb->format->cpp[0];
2519
2520         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2521         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2522                                  crtc_state->base.adjusted_mode.crtc_htotal,
2523                                  drm_rect_width(&plane_state->base.dst),
2524                                  cpp, mem_value);
2525         return min(method1, method2);
2526 }
2527
2528 /*
2529  * For both WM_PIPE and WM_LP.
2530  * mem_value must be in 0.1us units.
2531  */
2532 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2533                               const struct intel_plane_state *plane_state,
2534                               u32 mem_value)
2535 {
2536         int cpp;
2537
2538         if (mem_value == 0)
2539                 return U32_MAX;
2540
2541         if (!intel_wm_plane_visible(crtc_state, plane_state))
2542                 return 0;
2543
2544         cpp = plane_state->base.fb->format->cpp[0];
2545
2546         return ilk_wm_method2(crtc_state->pixel_rate,
2547                               crtc_state->base.adjusted_mode.crtc_htotal,
2548                               drm_rect_width(&plane_state->base.dst),
2549                               cpp, mem_value);
2550 }
2551
2552 /* Only for WM_LP. */
2553 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2554                               const struct intel_plane_state *plane_state,
2555                               u32 pri_val)
2556 {
2557         int cpp;
2558
2559         if (!intel_wm_plane_visible(crtc_state, plane_state))
2560                 return 0;
2561
2562         cpp = plane_state->base.fb->format->cpp[0];
2563
2564         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->base.dst), cpp);
2565 }
2566
2567 static unsigned int
2568 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2569 {
2570         if (INTEL_GEN(dev_priv) >= 8)
2571                 return 3072;
2572         else if (INTEL_GEN(dev_priv) >= 7)
2573                 return 768;
2574         else
2575                 return 512;
2576 }
2577
2578 static unsigned int
2579 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2580                      int level, bool is_sprite)
2581 {
2582         if (INTEL_GEN(dev_priv) >= 8)
2583                 /* BDW primary/sprite plane watermarks */
2584                 return level == 0 ? 255 : 2047;
2585         else if (INTEL_GEN(dev_priv) >= 7)
2586                 /* IVB/HSW primary/sprite plane watermarks */
2587                 return level == 0 ? 127 : 1023;
2588         else if (!is_sprite)
2589                 /* ILK/SNB primary plane watermarks */
2590                 return level == 0 ? 127 : 511;
2591         else
2592                 /* ILK/SNB sprite plane watermarks */
2593                 return level == 0 ? 63 : 255;
2594 }
2595
2596 static unsigned int
2597 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2598 {
2599         if (INTEL_GEN(dev_priv) >= 7)
2600                 return level == 0 ? 63 : 255;
2601         else
2602                 return level == 0 ? 31 : 63;
2603 }
2604
2605 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2606 {
2607         if (INTEL_GEN(dev_priv) >= 8)
2608                 return 31;
2609         else
2610                 return 15;
2611 }
2612
2613 /* Calculate the maximum primary/sprite plane watermark */
2614 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2615                                      int level,
2616                                      const struct intel_wm_config *config,
2617                                      enum intel_ddb_partitioning ddb_partitioning,
2618                                      bool is_sprite)
2619 {
2620         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2621
2622         /* if sprites aren't enabled, sprites get nothing */
2623         if (is_sprite && !config->sprites_enabled)
2624                 return 0;
2625
2626         /* HSW allows LP1+ watermarks even with multiple pipes */
2627         if (level == 0 || config->num_pipes_active > 1) {
2628                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2629
2630                 /*
2631                  * For some reason the non self refresh
2632                  * FIFO size is only half of the self
2633                  * refresh FIFO size on ILK/SNB.
2634                  */
2635                 if (INTEL_GEN(dev_priv) <= 6)
2636                         fifo_size /= 2;
2637         }
2638
2639         if (config->sprites_enabled) {
2640                 /* level 0 is always calculated with 1:1 split */
2641                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2642                         if (is_sprite)
2643                                 fifo_size *= 5;
2644                         fifo_size /= 6;
2645                 } else {
2646                         fifo_size /= 2;
2647                 }
2648         }
2649
2650         /* clamp to max that the registers can hold */
2651         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2652 }
2653
2654 /* Calculate the maximum cursor plane watermark */
2655 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2656                                       int level,
2657                                       const struct intel_wm_config *config)
2658 {
2659         /* HSW LP1+ watermarks w/ multiple pipes */
2660         if (level > 0 && config->num_pipes_active > 1)
2661                 return 64;
2662
2663         /* otherwise just report max that registers can hold */
2664         return ilk_cursor_wm_reg_max(dev_priv, level);
2665 }
2666
2667 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2668                                     int level,
2669                                     const struct intel_wm_config *config,
2670                                     enum intel_ddb_partitioning ddb_partitioning,
2671                                     struct ilk_wm_maximums *max)
2672 {
2673         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2674         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2675         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2676         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2677 }
2678
2679 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2680                                         int level,
2681                                         struct ilk_wm_maximums *max)
2682 {
2683         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2684         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2685         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2686         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2687 }
2688
2689 static bool ilk_validate_wm_level(int level,
2690                                   const struct ilk_wm_maximums *max,
2691                                   struct intel_wm_level *result)
2692 {
2693         bool ret;
2694
2695         /* already determined to be invalid? */
2696         if (!result->enable)
2697                 return false;
2698
2699         result->enable = result->pri_val <= max->pri &&
2700                          result->spr_val <= max->spr &&
2701                          result->cur_val <= max->cur;
2702
2703         ret = result->enable;
2704
2705         /*
2706          * HACK until we can pre-compute everything,
2707          * and thus fail gracefully if LP0 watermarks
2708          * are exceeded...
2709          */
2710         if (level == 0 && !result->enable) {
2711                 if (result->pri_val > max->pri)
2712                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2713                                       level, result->pri_val, max->pri);
2714                 if (result->spr_val > max->spr)
2715                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2716                                       level, result->spr_val, max->spr);
2717                 if (result->cur_val > max->cur)
2718                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2719                                       level, result->cur_val, max->cur);
2720
2721                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2722                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2723                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2724                 result->enable = true;
2725         }
2726
2727         return ret;
2728 }
2729
2730 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2731                                  const struct intel_crtc *intel_crtc,
2732                                  int level,
2733                                  struct intel_crtc_state *crtc_state,
2734                                  const struct intel_plane_state *pristate,
2735                                  const struct intel_plane_state *sprstate,
2736                                  const struct intel_plane_state *curstate,
2737                                  struct intel_wm_level *result)
2738 {
2739         u16 pri_latency = dev_priv->wm.pri_latency[level];
2740         u16 spr_latency = dev_priv->wm.spr_latency[level];
2741         u16 cur_latency = dev_priv->wm.cur_latency[level];
2742
2743         /* WM1+ latency values stored in 0.5us units */
2744         if (level > 0) {
2745                 pri_latency *= 5;
2746                 spr_latency *= 5;
2747                 cur_latency *= 5;
2748         }
2749
2750         if (pristate) {
2751                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2752                                                      pri_latency, level);
2753                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2754         }
2755
2756         if (sprstate)
2757                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2758
2759         if (curstate)
2760                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2761
2762         result->enable = true;
2763 }
2764
2765 static u32
2766 hsw_compute_linetime_wm(const struct intel_crtc_state *crtc_state)
2767 {
2768         const struct intel_atomic_state *intel_state =
2769                 to_intel_atomic_state(crtc_state->base.state);
2770         const struct drm_display_mode *adjusted_mode =
2771                 &crtc_state->base.adjusted_mode;
2772         u32 linetime, ips_linetime;
2773
2774         if (!crtc_state->base.active)
2775                 return 0;
2776         if (WARN_ON(adjusted_mode->crtc_clock == 0))
2777                 return 0;
2778         if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2779                 return 0;
2780
2781         /* The WM are computed with base on how long it takes to fill a single
2782          * row at the given clock rate, multiplied by 8.
2783          * */
2784         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2785                                      adjusted_mode->crtc_clock);
2786         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2787                                          intel_state->cdclk.logical.cdclk);
2788
2789         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2790                PIPE_WM_LINETIME_TIME(linetime);
2791 }
2792
2793 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2794                                   u16 wm[8])
2795 {
2796         struct intel_uncore *uncore = &dev_priv->uncore;
2797
2798         if (INTEL_GEN(dev_priv) >= 9) {
2799                 u32 val;
2800                 int ret, i;
2801                 int level, max_level = ilk_wm_max_level(dev_priv);
2802
2803                 /* read the first set of memory latencies[0:3] */
2804                 val = 0; /* data0 to be programmed to 0 for first set */
2805                 ret = sandybridge_pcode_read(dev_priv,
2806                                              GEN9_PCODE_READ_MEM_LATENCY,
2807                                              &val, NULL);
2808
2809                 if (ret) {
2810                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2811                         return;
2812                 }
2813
2814                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2815                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2816                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2817                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2818                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2819                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2820                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2821
2822                 /* read the second set of memory latencies[4:7] */
2823                 val = 1; /* data0 to be programmed to 1 for second set */
2824                 ret = sandybridge_pcode_read(dev_priv,
2825                                              GEN9_PCODE_READ_MEM_LATENCY,
2826                                              &val, NULL);
2827                 if (ret) {
2828                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2829                         return;
2830                 }
2831
2832                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2833                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2834                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2835                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2836                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2837                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2838                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2839
2840                 /*
2841                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2842                  * need to be disabled. We make sure to sanitize the values out
2843                  * of the punit to satisfy this requirement.
2844                  */
2845                 for (level = 1; level <= max_level; level++) {
2846                         if (wm[level] == 0) {
2847                                 for (i = level + 1; i <= max_level; i++)
2848                                         wm[i] = 0;
2849                                 break;
2850                         }
2851                 }
2852
2853                 /*
2854                  * WaWmMemoryReadLatency:skl+,glk
2855                  *
2856                  * punit doesn't take into account the read latency so we need
2857                  * to add 2us to the various latency levels we retrieve from the
2858                  * punit when level 0 response data us 0us.
2859                  */
2860                 if (wm[0] == 0) {
2861                         wm[0] += 2;
2862                         for (level = 1; level <= max_level; level++) {
2863                                 if (wm[level] == 0)
2864                                         break;
2865                                 wm[level] += 2;
2866                         }
2867                 }
2868
2869                 /*
2870                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2871                  * If we could not get dimm info enable this WA to prevent from
2872                  * any underrun. If not able to get Dimm info assume 16GB dimm
2873                  * to avoid any underrun.
2874                  */
2875                 if (dev_priv->dram_info.is_16gb_dimm)
2876                         wm[0] += 1;
2877
2878         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2879                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2880
2881                 wm[0] = (sskpd >> 56) & 0xFF;
2882                 if (wm[0] == 0)
2883                         wm[0] = sskpd & 0xF;
2884                 wm[1] = (sskpd >> 4) & 0xFF;
2885                 wm[2] = (sskpd >> 12) & 0xFF;
2886                 wm[3] = (sskpd >> 20) & 0x1FF;
2887                 wm[4] = (sskpd >> 32) & 0x1FF;
2888         } else if (INTEL_GEN(dev_priv) >= 6) {
2889                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2890
2891                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2892                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2893                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2894                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2895         } else if (INTEL_GEN(dev_priv) >= 5) {
2896                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2897
2898                 /* ILK primary LP0 latency is 700 ns */
2899                 wm[0] = 7;
2900                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2901                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2902         } else {
2903                 MISSING_CASE(INTEL_DEVID(dev_priv));
2904         }
2905 }
2906
2907 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2908                                        u16 wm[5])
2909 {
2910         /* ILK sprite LP0 latency is 1300 ns */
2911         if (IS_GEN(dev_priv, 5))
2912                 wm[0] = 13;
2913 }
2914
2915 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2916                                        u16 wm[5])
2917 {
2918         /* ILK cursor LP0 latency is 1300 ns */
2919         if (IS_GEN(dev_priv, 5))
2920                 wm[0] = 13;
2921 }
2922
2923 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2924 {
2925         /* how many WM levels are we expecting */
2926         if (INTEL_GEN(dev_priv) >= 9)
2927                 return 7;
2928         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2929                 return 4;
2930         else if (INTEL_GEN(dev_priv) >= 6)
2931                 return 3;
2932         else
2933                 return 2;
2934 }
2935
2936 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2937                                    const char *name,
2938                                    const u16 wm[8])
2939 {
2940         int level, max_level = ilk_wm_max_level(dev_priv);
2941
2942         for (level = 0; level <= max_level; level++) {
2943                 unsigned int latency = wm[level];
2944
2945                 if (latency == 0) {
2946                         DRM_DEBUG_KMS("%s WM%d latency not provided\n",
2947                                       name, level);
2948                         continue;
2949                 }
2950
2951                 /*
2952                  * - latencies are in us on gen9.
2953                  * - before then, WM1+ latency values are in 0.5us units
2954                  */
2955                 if (INTEL_GEN(dev_priv) >= 9)
2956                         latency *= 10;
2957                 else if (level > 0)
2958                         latency *= 5;
2959
2960                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2961                               name, level, wm[level],
2962                               latency / 10, latency % 10);
2963         }
2964 }
2965
2966 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2967                                     u16 wm[5], u16 min)
2968 {
2969         int level, max_level = ilk_wm_max_level(dev_priv);
2970
2971         if (wm[0] >= min)
2972                 return false;
2973
2974         wm[0] = max(wm[0], min);
2975         for (level = 1; level <= max_level; level++)
2976                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2977
2978         return true;
2979 }
2980
2981 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2982 {
2983         bool changed;
2984
2985         /*
2986          * The BIOS provided WM memory latency values are often
2987          * inadequate for high resolution displays. Adjust them.
2988          */
2989         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2990                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2991                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2992
2993         if (!changed)
2994                 return;
2995
2996         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2997         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
2998         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
2999         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3000 }
3001
3002 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3003 {
3004         /*
3005          * On some SNB machines (Thinkpad X220 Tablet at least)
3006          * LP3 usage can cause vblank interrupts to be lost.
3007          * The DEIIR bit will go high but it looks like the CPU
3008          * never gets interrupted.
3009          *
3010          * It's not clear whether other interrupt source could
3011          * be affected or if this is somehow limited to vblank
3012          * interrupts only. To play it safe we disable LP3
3013          * watermarks entirely.
3014          */
3015         if (dev_priv->wm.pri_latency[3] == 0 &&
3016             dev_priv->wm.spr_latency[3] == 0 &&
3017             dev_priv->wm.cur_latency[3] == 0)
3018                 return;
3019
3020         dev_priv->wm.pri_latency[3] = 0;
3021         dev_priv->wm.spr_latency[3] = 0;
3022         dev_priv->wm.cur_latency[3] = 0;
3023
3024         DRM_DEBUG_KMS("LP3 watermarks disabled due to potential for lost interrupts\n");
3025         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3026         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3027         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3028 }
3029
3030 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3031 {
3032         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3033
3034         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3035                sizeof(dev_priv->wm.pri_latency));
3036         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3037                sizeof(dev_priv->wm.pri_latency));
3038
3039         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3040         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3041
3042         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3043         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3044         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3045
3046         if (IS_GEN(dev_priv, 6)) {
3047                 snb_wm_latency_quirk(dev_priv);
3048                 snb_wm_lp3_irq_quirk(dev_priv);
3049         }
3050 }
3051
3052 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3053 {
3054         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3055         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3056 }
3057
3058 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3059                                  struct intel_pipe_wm *pipe_wm)
3060 {
3061         /* LP0 watermark maximums depend on this pipe alone */
3062         const struct intel_wm_config config = {
3063                 .num_pipes_active = 1,
3064                 .sprites_enabled = pipe_wm->sprites_enabled,
3065                 .sprites_scaled = pipe_wm->sprites_scaled,
3066         };
3067         struct ilk_wm_maximums max;
3068
3069         /* LP0 watermarks always use 1/2 DDB partitioning */
3070         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3071
3072         /* At least LP0 must be valid */
3073         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3074                 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3075                 return false;
3076         }
3077
3078         return true;
3079 }
3080
3081 /* Compute new watermarks for the pipe */
3082 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3083 {
3084         struct drm_atomic_state *state = crtc_state->base.state;
3085         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3086         struct intel_pipe_wm *pipe_wm;
3087         struct drm_device *dev = state->dev;
3088         const struct drm_i915_private *dev_priv = to_i915(dev);
3089         struct intel_plane *plane;
3090         const struct intel_plane_state *plane_state;
3091         const struct intel_plane_state *pristate = NULL;
3092         const struct intel_plane_state *sprstate = NULL;
3093         const struct intel_plane_state *curstate = NULL;
3094         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3095         struct ilk_wm_maximums max;
3096
3097         pipe_wm = &crtc_state->wm.ilk.optimal;
3098
3099         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3100                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3101                         pristate = plane_state;
3102                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3103                         sprstate = plane_state;
3104                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3105                         curstate = plane_state;
3106         }
3107
3108         pipe_wm->pipe_enabled = crtc_state->base.active;
3109         if (sprstate) {
3110                 pipe_wm->sprites_enabled = sprstate->base.visible;
3111                 pipe_wm->sprites_scaled = sprstate->base.visible &&
3112                         (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3113                          drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3114         }
3115
3116         usable_level = max_level;
3117
3118         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3119         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3120                 usable_level = 1;
3121
3122         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3123         if (pipe_wm->sprites_scaled)
3124                 usable_level = 0;
3125
3126         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3127         ilk_compute_wm_level(dev_priv, intel_crtc, 0, crtc_state,
3128                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3129
3130         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3131                 pipe_wm->linetime = hsw_compute_linetime_wm(crtc_state);
3132
3133         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3134                 return -EINVAL;
3135
3136         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3137
3138         for (level = 1; level <= usable_level; level++) {
3139                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3140
3141                 ilk_compute_wm_level(dev_priv, intel_crtc, level, crtc_state,
3142                                      pristate, sprstate, curstate, wm);
3143
3144                 /*
3145                  * Disable any watermark level that exceeds the
3146                  * register maximums since such watermarks are
3147                  * always invalid.
3148                  */
3149                 if (!ilk_validate_wm_level(level, &max, wm)) {
3150                         memset(wm, 0, sizeof(*wm));
3151                         break;
3152                 }
3153         }
3154
3155         return 0;
3156 }
3157
3158 /*
3159  * Build a set of 'intermediate' watermark values that satisfy both the old
3160  * state and the new state.  These can be programmed to the hardware
3161  * immediately.
3162  */
3163 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3164 {
3165         struct intel_crtc *intel_crtc = to_intel_crtc(newstate->base.crtc);
3166         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3167         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3168         struct intel_atomic_state *intel_state =
3169                 to_intel_atomic_state(newstate->base.state);
3170         const struct intel_crtc_state *oldstate =
3171                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3172         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3173         int level, max_level = ilk_wm_max_level(dev_priv);
3174
3175         /*
3176          * Start with the final, target watermarks, then combine with the
3177          * currently active watermarks to get values that are safe both before
3178          * and after the vblank.
3179          */
3180         *a = newstate->wm.ilk.optimal;
3181         if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base) ||
3182             intel_state->skip_intermediate_wm)
3183                 return 0;
3184
3185         a->pipe_enabled |= b->pipe_enabled;
3186         a->sprites_enabled |= b->sprites_enabled;
3187         a->sprites_scaled |= b->sprites_scaled;
3188
3189         for (level = 0; level <= max_level; level++) {
3190                 struct intel_wm_level *a_wm = &a->wm[level];
3191                 const struct intel_wm_level *b_wm = &b->wm[level];
3192
3193                 a_wm->enable &= b_wm->enable;
3194                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3195                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3196                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3197                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3198         }
3199
3200         /*
3201          * We need to make sure that these merged watermark values are
3202          * actually a valid configuration themselves.  If they're not,
3203          * there's no safe way to transition from the old state to
3204          * the new state, so we need to fail the atomic transaction.
3205          */
3206         if (!ilk_validate_pipe_wm(dev_priv, a))
3207                 return -EINVAL;
3208
3209         /*
3210          * If our intermediate WM are identical to the final WM, then we can
3211          * omit the post-vblank programming; only update if it's different.
3212          */
3213         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3214                 newstate->wm.need_postvbl_update = true;
3215
3216         return 0;
3217 }
3218
3219 /*
3220  * Merge the watermarks from all active pipes for a specific level.
3221  */
3222 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3223                                int level,
3224                                struct intel_wm_level *ret_wm)
3225 {
3226         const struct intel_crtc *intel_crtc;
3227
3228         ret_wm->enable = true;
3229
3230         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3231                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3232                 const struct intel_wm_level *wm = &active->wm[level];
3233
3234                 if (!active->pipe_enabled)
3235                         continue;
3236
3237                 /*
3238                  * The watermark values may have been used in the past,
3239                  * so we must maintain them in the registers for some
3240                  * time even if the level is now disabled.
3241                  */
3242                 if (!wm->enable)
3243                         ret_wm->enable = false;
3244
3245                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3246                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3247                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3248                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3249         }
3250 }
3251
3252 /*
3253  * Merge all low power watermarks for all active pipes.
3254  */
3255 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3256                          const struct intel_wm_config *config,
3257                          const struct ilk_wm_maximums *max,
3258                          struct intel_pipe_wm *merged)
3259 {
3260         int level, max_level = ilk_wm_max_level(dev_priv);
3261         int last_enabled_level = max_level;
3262
3263         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3264         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3265             config->num_pipes_active > 1)
3266                 last_enabled_level = 0;
3267
3268         /* ILK: FBC WM must be disabled always */
3269         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3270
3271         /* merge each WM1+ level */
3272         for (level = 1; level <= max_level; level++) {
3273                 struct intel_wm_level *wm = &merged->wm[level];
3274
3275                 ilk_merge_wm_level(dev_priv, level, wm);
3276
3277                 if (level > last_enabled_level)
3278                         wm->enable = false;
3279                 else if (!ilk_validate_wm_level(level, max, wm))
3280                         /* make sure all following levels get disabled */
3281                         last_enabled_level = level - 1;
3282
3283                 /*
3284                  * The spec says it is preferred to disable
3285                  * FBC WMs instead of disabling a WM level.
3286                  */
3287                 if (wm->fbc_val > max->fbc) {
3288                         if (wm->enable)
3289                                 merged->fbc_wm_enabled = false;
3290                         wm->fbc_val = 0;
3291                 }
3292         }
3293
3294         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3295         /*
3296          * FIXME this is racy. FBC might get enabled later.
3297          * What we should check here is whether FBC can be
3298          * enabled sometime later.
3299          */
3300         if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3301             intel_fbc_is_active(dev_priv)) {
3302                 for (level = 2; level <= max_level; level++) {
3303                         struct intel_wm_level *wm = &merged->wm[level];
3304
3305                         wm->enable = false;
3306                 }
3307         }
3308 }
3309
3310 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3311 {
3312         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3313         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3314 }
3315
3316 /* The value we need to program into the WM_LPx latency field */
3317 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3318                                       int level)
3319 {
3320         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3321                 return 2 * level;
3322         else
3323                 return dev_priv->wm.pri_latency[level];
3324 }
3325
3326 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3327                                    const struct intel_pipe_wm *merged,
3328                                    enum intel_ddb_partitioning partitioning,
3329                                    struct ilk_wm_values *results)
3330 {
3331         struct intel_crtc *intel_crtc;
3332         int level, wm_lp;
3333
3334         results->enable_fbc_wm = merged->fbc_wm_enabled;
3335         results->partitioning = partitioning;
3336
3337         /* LP1+ register values */
3338         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3339                 const struct intel_wm_level *r;
3340
3341                 level = ilk_wm_lp_to_level(wm_lp, merged);
3342
3343                 r = &merged->wm[level];
3344
3345                 /*
3346                  * Maintain the watermark values even if the level is
3347                  * disabled. Doing otherwise could cause underruns.
3348                  */
3349                 results->wm_lp[wm_lp - 1] =
3350                         (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3351                         (r->pri_val << WM1_LP_SR_SHIFT) |
3352                         r->cur_val;
3353
3354                 if (r->enable)
3355                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3356
3357                 if (INTEL_GEN(dev_priv) >= 8)
3358                         results->wm_lp[wm_lp - 1] |=
3359                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3360                 else
3361                         results->wm_lp[wm_lp - 1] |=
3362                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3363
3364                 /*
3365                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3366                  * level is disabled. Doing otherwise could cause underruns.
3367                  */
3368                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3369                         WARN_ON(wm_lp != 1);
3370                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3371                 } else
3372                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3373         }
3374
3375         /* LP0 register values */
3376         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3377                 enum pipe pipe = intel_crtc->pipe;
3378                 const struct intel_wm_level *r =
3379                         &intel_crtc->wm.active.ilk.wm[0];
3380
3381                 if (WARN_ON(!r->enable))
3382                         continue;
3383
3384                 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3385
3386                 results->wm_pipe[pipe] =
3387                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3388                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3389                         r->cur_val;
3390         }
3391 }
3392
3393 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3394  * case both are at the same level. Prefer r1 in case they're the same. */
3395 static struct intel_pipe_wm *
3396 ilk_find_best_result(struct drm_i915_private *dev_priv,
3397                      struct intel_pipe_wm *r1,
3398                      struct intel_pipe_wm *r2)
3399 {
3400         int level, max_level = ilk_wm_max_level(dev_priv);
3401         int level1 = 0, level2 = 0;
3402
3403         for (level = 1; level <= max_level; level++) {
3404                 if (r1->wm[level].enable)
3405                         level1 = level;
3406                 if (r2->wm[level].enable)
3407                         level2 = level;
3408         }
3409
3410         if (level1 == level2) {
3411                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3412                         return r2;
3413                 else
3414                         return r1;
3415         } else if (level1 > level2) {
3416                 return r1;
3417         } else {
3418                 return r2;
3419         }
3420 }
3421
3422 /* dirty bits used to track which watermarks need changes */
3423 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3424 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3425 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3426 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3427 #define WM_DIRTY_FBC (1 << 24)
3428 #define WM_DIRTY_DDB (1 << 25)
3429
3430 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3431                                          const struct ilk_wm_values *old,
3432                                          const struct ilk_wm_values *new)
3433 {
3434         unsigned int dirty = 0;
3435         enum pipe pipe;
3436         int wm_lp;
3437
3438         for_each_pipe(dev_priv, pipe) {
3439                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3440                         dirty |= WM_DIRTY_LINETIME(pipe);
3441                         /* Must disable LP1+ watermarks too */
3442                         dirty |= WM_DIRTY_LP_ALL;
3443                 }
3444
3445                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3446                         dirty |= WM_DIRTY_PIPE(pipe);
3447                         /* Must disable LP1+ watermarks too */
3448                         dirty |= WM_DIRTY_LP_ALL;
3449                 }
3450         }
3451
3452         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3453                 dirty |= WM_DIRTY_FBC;
3454                 /* Must disable LP1+ watermarks too */
3455                 dirty |= WM_DIRTY_LP_ALL;
3456         }
3457
3458         if (old->partitioning != new->partitioning) {
3459                 dirty |= WM_DIRTY_DDB;
3460                 /* Must disable LP1+ watermarks too */
3461                 dirty |= WM_DIRTY_LP_ALL;
3462         }
3463
3464         /* LP1+ watermarks already deemed dirty, no need to continue */
3465         if (dirty & WM_DIRTY_LP_ALL)
3466                 return dirty;
3467
3468         /* Find the lowest numbered LP1+ watermark in need of an update... */
3469         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3470                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3471                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3472                         break;
3473         }
3474
3475         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3476         for (; wm_lp <= 3; wm_lp++)
3477                 dirty |= WM_DIRTY_LP(wm_lp);
3478
3479         return dirty;
3480 }
3481
3482 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3483                                unsigned int dirty)
3484 {
3485         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3486         bool changed = false;
3487
3488         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3489                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3490                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3491                 changed = true;
3492         }
3493         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3494                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3495                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3496                 changed = true;
3497         }
3498         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3499                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3500                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3501                 changed = true;
3502         }
3503
3504         /*
3505          * Don't touch WM1S_LP_EN here.
3506          * Doing so could cause underruns.
3507          */
3508
3509         return changed;
3510 }
3511
3512 /*
3513  * The spec says we shouldn't write when we don't need, because every write
3514  * causes WMs to be re-evaluated, expending some power.
3515  */
3516 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3517                                 struct ilk_wm_values *results)
3518 {
3519         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3520         unsigned int dirty;
3521         u32 val;
3522
3523         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3524         if (!dirty)
3525                 return;
3526
3527         _ilk_disable_lp_wm(dev_priv, dirty);
3528
3529         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3530                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3531         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3532                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3533         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3534                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3535
3536         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3537                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3538         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3539                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3540         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3541                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3542
3543         if (dirty & WM_DIRTY_DDB) {
3544                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3545                         val = I915_READ(WM_MISC);
3546                         if (results->partitioning == INTEL_DDB_PART_1_2)
3547                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3548                         else
3549                                 val |= WM_MISC_DATA_PARTITION_5_6;
3550                         I915_WRITE(WM_MISC, val);
3551                 } else {
3552                         val = I915_READ(DISP_ARB_CTL2);
3553                         if (results->partitioning == INTEL_DDB_PART_1_2)
3554                                 val &= ~DISP_DATA_PARTITION_5_6;
3555                         else
3556                                 val |= DISP_DATA_PARTITION_5_6;
3557                         I915_WRITE(DISP_ARB_CTL2, val);
3558                 }
3559         }
3560
3561         if (dirty & WM_DIRTY_FBC) {
3562                 val = I915_READ(DISP_ARB_CTL);
3563                 if (results->enable_fbc_wm)
3564                         val &= ~DISP_FBC_WM_DIS;
3565                 else
3566                         val |= DISP_FBC_WM_DIS;
3567                 I915_WRITE(DISP_ARB_CTL, val);
3568         }
3569
3570         if (dirty & WM_DIRTY_LP(1) &&
3571             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3572                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3573
3574         if (INTEL_GEN(dev_priv) >= 7) {
3575                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3576                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3577                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3578                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3579         }
3580
3581         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3582                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3583         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3584                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3585         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3586                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3587
3588         dev_priv->wm.hw = *results;
3589 }
3590
3591 bool ilk_disable_lp_wm(struct drm_device *dev)
3592 {
3593         struct drm_i915_private *dev_priv = to_i915(dev);
3594
3595         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3596 }
3597
3598 static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3599 {
3600         u8 enabled_slices;
3601
3602         /* Slice 1 will always be enabled */
3603         enabled_slices = 1;
3604
3605         /* Gen prior to GEN11 have only one DBuf slice */
3606         if (INTEL_GEN(dev_priv) < 11)
3607                 return enabled_slices;
3608
3609         /*
3610          * FIXME: for now we'll only ever use 1 slice; pretend that we have
3611          * only that 1 slice enabled until we have a proper way for on-demand
3612          * toggling of the second slice.
3613          */
3614         if (0 && I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3615                 enabled_slices++;
3616
3617         return enabled_slices;
3618 }
3619
3620 /*
3621  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3622  * so assume we'll always need it in order to avoid underruns.
3623  */
3624 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3625 {
3626         return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3627 }
3628
3629 static bool
3630 intel_has_sagv(struct drm_i915_private *dev_priv)
3631 {
3632         /* HACK! */
3633         if (IS_GEN(dev_priv, 12))
3634                 return false;
3635
3636         return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3637                 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3638 }
3639
3640 static void
3641 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3642 {
3643         if (INTEL_GEN(dev_priv) >= 12) {
3644                 u32 val = 0;
3645                 int ret;
3646
3647                 ret = sandybridge_pcode_read(dev_priv,
3648                                              GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3649                                              &val, NULL);
3650                 if (!ret) {
3651                         dev_priv->sagv_block_time_us = val;
3652                         return;
3653                 }
3654
3655                 DRM_DEBUG_DRIVER("Couldn't read SAGV block time!\n");
3656         } else if (IS_GEN(dev_priv, 11)) {
3657                 dev_priv->sagv_block_time_us = 10;
3658                 return;
3659         } else if (IS_GEN(dev_priv, 10)) {
3660                 dev_priv->sagv_block_time_us = 20;
3661                 return;
3662         } else if (IS_GEN(dev_priv, 9)) {
3663                 dev_priv->sagv_block_time_us = 30;
3664                 return;
3665         } else {
3666                 MISSING_CASE(INTEL_GEN(dev_priv));
3667         }
3668
3669         /* Default to an unusable block time */
3670         dev_priv->sagv_block_time_us = -1;
3671 }
3672
3673 /*
3674  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3675  * depending on power and performance requirements. The display engine access
3676  * to system memory is blocked during the adjustment time. Because of the
3677  * blocking time, having this enabled can cause full system hangs and/or pipe
3678  * underruns if we don't meet all of the following requirements:
3679  *
3680  *  - <= 1 pipe enabled
3681  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3682  *  - We're not using an interlaced display configuration
3683  */
3684 int
3685 intel_enable_sagv(struct drm_i915_private *dev_priv)
3686 {
3687         int ret;
3688
3689         if (!intel_has_sagv(dev_priv))
3690                 return 0;
3691
3692         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3693                 return 0;
3694
3695         DRM_DEBUG_KMS("Enabling SAGV\n");
3696         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3697                                       GEN9_SAGV_ENABLE);
3698
3699         /* We don't need to wait for SAGV when enabling */
3700
3701         /*
3702          * Some skl systems, pre-release machines in particular,
3703          * don't actually have SAGV.
3704          */
3705         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3706                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3707                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3708                 return 0;
3709         } else if (ret < 0) {
3710                 DRM_ERROR("Failed to enable SAGV\n");
3711                 return ret;
3712         }
3713
3714         dev_priv->sagv_status = I915_SAGV_ENABLED;
3715         return 0;
3716 }
3717
3718 int
3719 intel_disable_sagv(struct drm_i915_private *dev_priv)
3720 {
3721         int ret;
3722
3723         if (!intel_has_sagv(dev_priv))
3724                 return 0;
3725
3726         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3727                 return 0;
3728
3729         DRM_DEBUG_KMS("Disabling SAGV\n");
3730         /* bspec says to keep retrying for at least 1 ms */
3731         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3732                                 GEN9_SAGV_DISABLE,
3733                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3734                                 1);
3735         /*
3736          * Some skl systems, pre-release machines in particular,
3737          * don't actually have SAGV.
3738          */
3739         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3740                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3741                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3742                 return 0;
3743         } else if (ret < 0) {
3744                 DRM_ERROR("Failed to disable SAGV (%d)\n", ret);
3745                 return ret;
3746         }
3747
3748         dev_priv->sagv_status = I915_SAGV_DISABLED;
3749         return 0;
3750 }
3751
3752 bool intel_can_enable_sagv(struct intel_atomic_state *state)
3753 {
3754         struct drm_device *dev = state->base.dev;
3755         struct drm_i915_private *dev_priv = to_i915(dev);
3756         struct intel_crtc *crtc;
3757         struct intel_plane *plane;
3758         struct intel_crtc_state *crtc_state;
3759         enum pipe pipe;
3760         int level, latency;
3761
3762         if (!intel_has_sagv(dev_priv))
3763                 return false;
3764
3765         /*
3766          * If there are no active CRTCs, no additional checks need be performed
3767          */
3768         if (hweight8(state->active_pipes) == 0)
3769                 return true;
3770
3771         /*
3772          * SKL+ workaround: bspec recommends we disable SAGV when we have
3773          * more then one pipe enabled
3774          */
3775         if (hweight8(state->active_pipes) > 1)
3776                 return false;
3777
3778         /* Since we're now guaranteed to only have one active CRTC... */
3779         pipe = ffs(state->active_pipes) - 1;
3780         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3781         crtc_state = to_intel_crtc_state(crtc->base.state);
3782
3783         if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3784                 return false;
3785
3786         for_each_intel_plane_on_crtc(dev, crtc, plane) {
3787                 struct skl_plane_wm *wm =
3788                         &crtc_state->wm.skl.optimal.planes[plane->id];
3789
3790                 /* Skip this plane if it's not enabled */
3791                 if (!wm->wm[0].plane_en)
3792                         continue;
3793
3794                 /* Find the highest enabled wm level for this plane */
3795                 for (level = ilk_wm_max_level(dev_priv);
3796                      !wm->wm[level].plane_en; --level)
3797                      { }
3798
3799                 latency = dev_priv->wm.skl_latency[level];
3800
3801                 if (skl_needs_memory_bw_wa(dev_priv) &&
3802                     plane->base.state->fb->modifier ==
3803                     I915_FORMAT_MOD_X_TILED)
3804                         latency += 15;
3805
3806                 /*
3807                  * If any of the planes on this pipe don't enable wm levels that
3808                  * incur memory latencies higher than sagv_block_time_us we
3809                  * can't enable SAGV.
3810                  */
3811                 if (latency < dev_priv->sagv_block_time_us)
3812                         return false;
3813         }
3814
3815         return true;
3816 }
3817
3818 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv,
3819                               const struct intel_crtc_state *crtc_state,
3820                               const u64 total_data_rate,
3821                               const int num_active,
3822                               struct skl_ddb_allocation *ddb)
3823 {
3824         const struct drm_display_mode *adjusted_mode;
3825         u64 total_data_bw;
3826         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3827
3828         WARN_ON(ddb_size == 0);
3829
3830         if (INTEL_GEN(dev_priv) < 11)
3831                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
3832
3833         adjusted_mode = &crtc_state->base.adjusted_mode;
3834         total_data_bw = total_data_rate * drm_mode_vrefresh(adjusted_mode);
3835
3836         /*
3837          * 12GB/s is maximum BW supported by single DBuf slice.
3838          *
3839          * FIXME dbuf slice code is broken:
3840          * - must wait for planes to stop using the slice before powering it off
3841          * - plane straddling both slices is illegal in multi-pipe scenarios
3842          * - should validate we stay within the hw bandwidth limits
3843          */
3844         if (0 && (num_active > 1 || total_data_bw >= GBps(12))) {
3845                 ddb->enabled_slices = 2;
3846         } else {
3847                 ddb->enabled_slices = 1;
3848                 ddb_size /= 2;
3849         }
3850
3851         return ddb_size;
3852 }
3853
3854 static void
3855 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
3856                                    const struct intel_crtc_state *crtc_state,
3857                                    const u64 total_data_rate,
3858                                    struct skl_ddb_allocation *ddb,
3859                                    struct skl_ddb_entry *alloc, /* out */
3860                                    int *num_active /* out */)
3861 {
3862         struct drm_atomic_state *state = crtc_state->base.state;
3863         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3864         struct drm_crtc *for_crtc = crtc_state->base.crtc;
3865         const struct intel_crtc *crtc;
3866         u32 pipe_width = 0, total_width = 0, width_before_pipe = 0;
3867         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
3868         u16 ddb_size;
3869         u32 i;
3870
3871         if (WARN_ON(!state) || !crtc_state->base.active) {
3872                 alloc->start = 0;
3873                 alloc->end = 0;
3874                 *num_active = hweight8(dev_priv->active_pipes);
3875                 return;
3876         }
3877
3878         if (intel_state->active_pipe_changes)
3879                 *num_active = hweight8(intel_state->active_pipes);
3880         else
3881                 *num_active = hweight8(dev_priv->active_pipes);
3882
3883         ddb_size = intel_get_ddb_size(dev_priv, crtc_state, total_data_rate,
3884                                       *num_active, ddb);
3885
3886         /*
3887          * If the state doesn't change the active CRTC's or there is no
3888          * modeset request, then there's no need to recalculate;
3889          * the existing pipe allocation limits should remain unchanged.
3890          * Note that we're safe from racing commits since any racing commit
3891          * that changes the active CRTC list or do modeset would need to
3892          * grab _all_ crtc locks, including the one we currently hold.
3893          */
3894         if (!intel_state->active_pipe_changes && !intel_state->modeset) {
3895                 /*
3896                  * alloc may be cleared by clear_intel_crtc_state,
3897                  * copy from old state to be sure
3898                  */
3899                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3900                 return;
3901         }
3902
3903         /*
3904          * Watermark/ddb requirement highly depends upon width of the
3905          * framebuffer, So instead of allocating DDB equally among pipes
3906          * distribute DDB based on resolution/width of the display.
3907          */
3908         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3909                 const struct drm_display_mode *adjusted_mode =
3910                         &crtc_state->base.adjusted_mode;
3911                 enum pipe pipe = crtc->pipe;
3912                 int hdisplay, vdisplay;
3913
3914                 if (!crtc_state->base.enable)
3915                         continue;
3916
3917                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
3918                 total_width += hdisplay;
3919
3920                 if (pipe < for_pipe)
3921                         width_before_pipe += hdisplay;
3922                 else if (pipe == for_pipe)
3923                         pipe_width = hdisplay;
3924         }
3925
3926         alloc->start = ddb_size * width_before_pipe / total_width;
3927         alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width;
3928 }
3929
3930 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
3931                                  int width, const struct drm_format_info *format,
3932                                  u64 modifier, unsigned int rotation,
3933                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
3934                                  int color_plane);
3935 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
3936                                  int level,
3937                                  const struct skl_wm_params *wp,
3938                                  const struct skl_wm_level *result_prev,
3939                                  struct skl_wm_level *result /* out */);
3940
3941 static unsigned int
3942 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
3943                       int num_active)
3944 {
3945         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3946         int level, max_level = ilk_wm_max_level(dev_priv);
3947         struct skl_wm_level wm = {};
3948         int ret, min_ddb_alloc = 0;
3949         struct skl_wm_params wp;
3950
3951         ret = skl_compute_wm_params(crtc_state, 256,
3952                                     drm_format_info(DRM_FORMAT_ARGB8888),
3953                                     DRM_FORMAT_MOD_LINEAR,
3954                                     DRM_MODE_ROTATE_0,
3955                                     crtc_state->pixel_rate, &wp, 0);
3956         WARN_ON(ret);
3957
3958         for (level = 0; level <= max_level; level++) {
3959                 skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm);
3960                 if (wm.min_ddb_alloc == U16_MAX)
3961                         break;
3962
3963                 min_ddb_alloc = wm.min_ddb_alloc;
3964         }
3965
3966         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
3967 }
3968
3969 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3970                                        struct skl_ddb_entry *entry, u32 reg)
3971 {
3972
3973         entry->start = reg & DDB_ENTRY_MASK;
3974         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
3975
3976         if (entry->end)
3977                 entry->end += 1;
3978 }
3979
3980 static void
3981 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3982                            const enum pipe pipe,
3983                            const enum plane_id plane_id,
3984                            struct skl_ddb_entry *ddb_y,
3985                            struct skl_ddb_entry *ddb_uv)
3986 {
3987         u32 val, val2;
3988         u32 fourcc = 0;
3989
3990         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
3991         if (plane_id == PLANE_CURSOR) {
3992                 val = I915_READ(CUR_BUF_CFG(pipe));
3993                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
3994                 return;
3995         }
3996
3997         val = I915_READ(PLANE_CTL(pipe, plane_id));
3998
3999         /* No DDB allocated for disabled planes */
4000         if (val & PLANE_CTL_ENABLE)
4001                 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4002                                               val & PLANE_CTL_ORDER_RGBX,
4003                                               val & PLANE_CTL_ALPHA_MASK);
4004
4005         if (INTEL_GEN(dev_priv) >= 11) {
4006                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4007                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4008         } else {
4009                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4010                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4011
4012                 if (fourcc &&
4013                     drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4014                         swap(val, val2);
4015
4016                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4017                 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4018         }
4019 }
4020
4021 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4022                                struct skl_ddb_entry *ddb_y,
4023                                struct skl_ddb_entry *ddb_uv)
4024 {
4025         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4026         enum intel_display_power_domain power_domain;
4027         enum pipe pipe = crtc->pipe;
4028         intel_wakeref_t wakeref;
4029         enum plane_id plane_id;
4030
4031         power_domain = POWER_DOMAIN_PIPE(pipe);
4032         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4033         if (!wakeref)
4034                 return;
4035
4036         for_each_plane_id_on_crtc(crtc, plane_id)
4037                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4038                                            plane_id,
4039                                            &ddb_y[plane_id],
4040                                            &ddb_uv[plane_id]);
4041
4042         intel_display_power_put(dev_priv, power_domain, wakeref);
4043 }
4044
4045 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
4046                           struct skl_ddb_allocation *ddb /* out */)
4047 {
4048         ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
4049 }
4050
4051 /*
4052  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4053  * The bspec defines downscale amount as:
4054  *
4055  * """
4056  * Horizontal down scale amount = maximum[1, Horizontal source size /
4057  *                                           Horizontal destination size]
4058  * Vertical down scale amount = maximum[1, Vertical source size /
4059  *                                         Vertical destination size]
4060  * Total down scale amount = Horizontal down scale amount *
4061  *                           Vertical down scale amount
4062  * """
4063  *
4064  * Return value is provided in 16.16 fixed point form to retain fractional part.
4065  * Caller should take care of dividing & rounding off the value.
4066  */
4067 static uint_fixed_16_16_t
4068 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4069                            const struct intel_plane_state *plane_state)
4070 {
4071         u32 src_w, src_h, dst_w, dst_h;
4072         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4073         uint_fixed_16_16_t downscale_h, downscale_w;
4074
4075         if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4076                 return u32_to_fixed16(0);
4077
4078         /*
4079          * Src coordinates are already rotated by 270 degrees for
4080          * the 90/270 degree plane rotation cases (to match the
4081          * GTT mapping), hence no need to account for rotation here.
4082          *
4083          * n.b., src is 16.16 fixed point, dst is whole integer.
4084          */
4085         src_w = drm_rect_width(&plane_state->base.src) >> 16;
4086         src_h = drm_rect_height(&plane_state->base.src) >> 16;
4087         dst_w = drm_rect_width(&plane_state->base.dst);
4088         dst_h = drm_rect_height(&plane_state->base.dst);
4089
4090         fp_w_ratio = div_fixed16(src_w, dst_w);
4091         fp_h_ratio = div_fixed16(src_h, dst_h);
4092         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4093         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4094
4095         return mul_fixed16(downscale_w, downscale_h);
4096 }
4097
4098 static u64
4099 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4100                              const struct intel_plane_state *plane_state,
4101                              int color_plane)
4102 {
4103         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4104         const struct drm_framebuffer *fb = plane_state->base.fb;
4105         u32 data_rate;
4106         u32 width = 0, height = 0;
4107         uint_fixed_16_16_t down_scale_amount;
4108         u64 rate;
4109
4110         if (!plane_state->base.visible)
4111                 return 0;
4112
4113         if (plane->id == PLANE_CURSOR)
4114                 return 0;
4115
4116         if (color_plane == 1 &&
4117             !drm_format_info_is_yuv_semiplanar(fb->format))
4118                 return 0;
4119
4120         /*
4121          * Src coordinates are already rotated by 270 degrees for
4122          * the 90/270 degree plane rotation cases (to match the
4123          * GTT mapping), hence no need to account for rotation here.
4124          */
4125         width = drm_rect_width(&plane_state->base.src) >> 16;
4126         height = drm_rect_height(&plane_state->base.src) >> 16;
4127
4128         /* UV plane does 1/2 pixel sub-sampling */
4129         if (color_plane == 1) {
4130                 width /= 2;
4131                 height /= 2;
4132         }
4133
4134         data_rate = width * height;
4135
4136         down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4137
4138         rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4139
4140         rate *= fb->format->cpp[color_plane];
4141         return rate;
4142 }
4143
4144 static u64
4145 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4146                                  u64 *plane_data_rate,
4147                                  u64 *uv_plane_data_rate)
4148 {
4149         struct drm_atomic_state *state = crtc_state->base.state;
4150         struct intel_plane *plane;
4151         const struct intel_plane_state *plane_state;
4152         u64 total_data_rate = 0;
4153
4154         if (WARN_ON(!state))
4155                 return 0;
4156
4157         /* Calculate and cache data rate for each plane */
4158         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4159                 enum plane_id plane_id = plane->id;
4160                 u64 rate;
4161
4162                 /* packed/y */
4163                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4164                 plane_data_rate[plane_id] = rate;
4165                 total_data_rate += rate;
4166
4167                 /* uv-plane */
4168                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4169                 uv_plane_data_rate[plane_id] = rate;
4170                 total_data_rate += rate;
4171         }
4172
4173         return total_data_rate;
4174 }
4175
4176 static u64
4177 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4178                                  u64 *plane_data_rate)
4179 {
4180         struct intel_plane *plane;
4181         const struct intel_plane_state *plane_state;
4182         u64 total_data_rate = 0;
4183
4184         if (WARN_ON(!crtc_state->base.state))
4185                 return 0;
4186
4187         /* Calculate and cache data rate for each plane */
4188         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4189                 enum plane_id plane_id = plane->id;
4190                 u64 rate;
4191
4192                 if (!plane_state->planar_linked_plane) {
4193                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4194                         plane_data_rate[plane_id] = rate;
4195                         total_data_rate += rate;
4196                 } else {
4197                         enum plane_id y_plane_id;
4198
4199                         /*
4200                          * The slave plane might not iterate in
4201                          * intel_atomic_crtc_state_for_each_plane_state(),
4202                          * and needs the master plane state which may be
4203                          * NULL if we try get_new_plane_state(), so we
4204                          * always calculate from the master.
4205                          */
4206                         if (plane_state->planar_slave)
4207                                 continue;
4208
4209                         /* Y plane rate is calculated on the slave */
4210                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4211                         y_plane_id = plane_state->planar_linked_plane->id;
4212                         plane_data_rate[y_plane_id] = rate;
4213                         total_data_rate += rate;
4214
4215                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4216                         plane_data_rate[plane_id] = rate;
4217                         total_data_rate += rate;
4218                 }
4219         }
4220
4221         return total_data_rate;
4222 }
4223
4224 static int
4225 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state,
4226                       struct skl_ddb_allocation *ddb /* out */)
4227 {
4228         struct drm_atomic_state *state = crtc_state->base.state;
4229         struct drm_crtc *crtc = crtc_state->base.crtc;
4230         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
4231         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4232         struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4233         u16 alloc_size, start = 0;
4234         u16 total[I915_MAX_PLANES] = {};
4235         u16 uv_total[I915_MAX_PLANES] = {};
4236         u64 total_data_rate;
4237         enum plane_id plane_id;
4238         int num_active;
4239         u64 plane_data_rate[I915_MAX_PLANES] = {};
4240         u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4241         u32 blocks;
4242         int level;
4243
4244         /* Clear the partitioning for disabled planes. */
4245         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4246         memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4247
4248         if (WARN_ON(!state))
4249                 return 0;
4250
4251         if (!crtc_state->base.active) {
4252                 alloc->start = alloc->end = 0;
4253                 return 0;
4254         }
4255
4256         if (INTEL_GEN(dev_priv) >= 11)
4257                 total_data_rate =
4258                         icl_get_total_relative_data_rate(crtc_state,
4259                                                          plane_data_rate);
4260         else
4261                 total_data_rate =
4262                         skl_get_total_relative_data_rate(crtc_state,
4263                                                          plane_data_rate,
4264                                                          uv_plane_data_rate);
4265
4266
4267         skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate,
4268                                            ddb, alloc, &num_active);
4269         alloc_size = skl_ddb_entry_size(alloc);
4270         if (alloc_size == 0)
4271                 return 0;
4272
4273         /* Allocate fixed number of blocks for cursor. */
4274         total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4275         alloc_size -= total[PLANE_CURSOR];
4276         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4277                 alloc->end - total[PLANE_CURSOR];
4278         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4279
4280         if (total_data_rate == 0)
4281                 return 0;
4282
4283         /*
4284          * Find the highest watermark level for which we can satisfy the block
4285          * requirement of active planes.
4286          */
4287         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4288                 blocks = 0;
4289                 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4290                         const struct skl_plane_wm *wm =
4291                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4292
4293                         if (plane_id == PLANE_CURSOR) {
4294                                 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4295                                         WARN_ON(wm->wm[level].min_ddb_alloc != U16_MAX);
4296                                         blocks = U32_MAX;
4297                                         break;
4298                                 }
4299                                 continue;
4300                         }
4301
4302                         blocks += wm->wm[level].min_ddb_alloc;
4303                         blocks += wm->uv_wm[level].min_ddb_alloc;
4304                 }
4305
4306                 if (blocks <= alloc_size) {
4307                         alloc_size -= blocks;
4308                         break;
4309                 }
4310         }
4311
4312         if (level < 0) {
4313                 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4314                 DRM_DEBUG_KMS("minimum required %d/%d\n", blocks,
4315                               alloc_size);
4316                 return -EINVAL;
4317         }
4318
4319         /*
4320          * Grant each plane the blocks it requires at the highest achievable
4321          * watermark level, plus an extra share of the leftover blocks
4322          * proportional to its relative data rate.
4323          */
4324         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4325                 const struct skl_plane_wm *wm =
4326                         &crtc_state->wm.skl.optimal.planes[plane_id];
4327                 u64 rate;
4328                 u16 extra;
4329
4330                 if (plane_id == PLANE_CURSOR)
4331                         continue;
4332
4333                 /*
4334                  * We've accounted for all active planes; remaining planes are
4335                  * all disabled.
4336                  */
4337                 if (total_data_rate == 0)
4338                         break;
4339
4340                 rate = plane_data_rate[plane_id];
4341                 extra = min_t(u16, alloc_size,
4342                               DIV64_U64_ROUND_UP(alloc_size * rate,
4343                                                  total_data_rate));
4344                 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4345                 alloc_size -= extra;
4346                 total_data_rate -= rate;
4347
4348                 if (total_data_rate == 0)
4349                         break;
4350
4351                 rate = uv_plane_data_rate[plane_id];
4352                 extra = min_t(u16, alloc_size,
4353                               DIV64_U64_ROUND_UP(alloc_size * rate,
4354                                                  total_data_rate));
4355                 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4356                 alloc_size -= extra;
4357                 total_data_rate -= rate;
4358         }
4359         WARN_ON(alloc_size != 0 || total_data_rate != 0);
4360
4361         /* Set the actual DDB start/end points for each plane */
4362         start = alloc->start;
4363         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4364                 struct skl_ddb_entry *plane_alloc =
4365                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
4366                 struct skl_ddb_entry *uv_plane_alloc =
4367                         &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4368
4369                 if (plane_id == PLANE_CURSOR)
4370                         continue;
4371
4372                 /* Gen11+ uses a separate plane for UV watermarks */
4373                 WARN_ON(INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4374
4375                 /* Leave disabled planes at (0,0) */
4376                 if (total[plane_id]) {
4377                         plane_alloc->start = start;
4378                         start += total[plane_id];
4379                         plane_alloc->end = start;
4380                 }
4381
4382                 if (uv_total[plane_id]) {
4383                         uv_plane_alloc->start = start;
4384                         start += uv_total[plane_id];
4385                         uv_plane_alloc->end = start;
4386                 }
4387         }
4388
4389         /*
4390          * When we calculated watermark values we didn't know how high
4391          * of a level we'd actually be able to hit, so we just marked
4392          * all levels as "enabled."  Go back now and disable the ones
4393          * that aren't actually possible.
4394          */
4395         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4396                 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4397                         struct skl_plane_wm *wm =
4398                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4399
4400                         /*
4401                          * We only disable the watermarks for each plane if
4402                          * they exceed the ddb allocation of said plane. This
4403                          * is done so that we don't end up touching cursor
4404                          * watermarks needlessly when some other plane reduces
4405                          * our max possible watermark level.
4406                          *
4407                          * Bspec has this to say about the PLANE_WM enable bit:
4408                          * "All the watermarks at this level for all enabled
4409                          *  planes must be enabled before the level will be used."
4410                          * So this is actually safe to do.
4411                          */
4412                         if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
4413                             wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
4414                                 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
4415
4416                         /*
4417                          * Wa_1408961008:icl, ehl
4418                          * Underruns with WM1+ disabled
4419                          */
4420                         if (IS_GEN(dev_priv, 11) &&
4421                             level == 1 && wm->wm[0].plane_en) {
4422                                 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
4423                                 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
4424                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4425                         }
4426                 }
4427         }
4428
4429         /*
4430          * Go back and disable the transition watermark if it turns out we
4431          * don't have enough DDB blocks for it.
4432          */
4433         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4434                 struct skl_plane_wm *wm =
4435                         &crtc_state->wm.skl.optimal.planes[plane_id];
4436
4437                 if (wm->trans_wm.plane_res_b >= total[plane_id])
4438                         memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
4439         }
4440
4441         return 0;
4442 }
4443
4444 /*
4445  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4446  * for the read latency) and cpp should always be <= 8, so that
4447  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4448  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4449 */
4450 static uint_fixed_16_16_t
4451 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
4452                u8 cpp, u32 latency, u32 dbuf_block_size)
4453 {
4454         u32 wm_intermediate_val;
4455         uint_fixed_16_16_t ret;
4456
4457         if (latency == 0)
4458                 return FP_16_16_MAX;
4459
4460         wm_intermediate_val = latency * pixel_rate * cpp;
4461         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4462
4463         if (INTEL_GEN(dev_priv) >= 10)
4464                 ret = add_fixed16_u32(ret, 1);
4465
4466         return ret;
4467 }
4468
4469 static uint_fixed_16_16_t
4470 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
4471                uint_fixed_16_16_t plane_blocks_per_line)
4472 {
4473         u32 wm_intermediate_val;
4474         uint_fixed_16_16_t ret;
4475
4476         if (latency == 0)
4477                 return FP_16_16_MAX;
4478
4479         wm_intermediate_val = latency * pixel_rate;
4480         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4481                                            pipe_htotal * 1000);
4482         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4483         return ret;
4484 }
4485
4486 static uint_fixed_16_16_t
4487 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
4488 {
4489         u32 pixel_rate;
4490         u32 crtc_htotal;
4491         uint_fixed_16_16_t linetime_us;
4492
4493         if (!crtc_state->base.active)
4494                 return u32_to_fixed16(0);
4495
4496         pixel_rate = crtc_state->pixel_rate;
4497
4498         if (WARN_ON(pixel_rate == 0))
4499                 return u32_to_fixed16(0);
4500
4501         crtc_htotal = crtc_state->base.adjusted_mode.crtc_htotal;
4502         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4503
4504         return linetime_us;
4505 }
4506
4507 static u32
4508 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
4509                               const struct intel_plane_state *plane_state)
4510 {
4511         u64 adjusted_pixel_rate;
4512         uint_fixed_16_16_t downscale_amount;
4513
4514         /* Shouldn't reach here on disabled planes... */
4515         if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4516                 return 0;
4517
4518         /*
4519          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
4520          * with additional adjustments for plane-specific scaling.
4521          */
4522         adjusted_pixel_rate = crtc_state->pixel_rate;
4523         downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4524
4525         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4526                                             downscale_amount);
4527 }
4528
4529 static int
4530 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4531                       int width, const struct drm_format_info *format,
4532                       u64 modifier, unsigned int rotation,
4533                       u32 plane_pixel_rate, struct skl_wm_params *wp,
4534                       int color_plane)
4535 {
4536         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4537         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4538         u32 interm_pbpl;
4539
4540         /* only planar format has two planes */
4541         if (color_plane == 1 && !drm_format_info_is_yuv_semiplanar(format)) {
4542                 DRM_DEBUG_KMS("Non planar format have single plane\n");
4543                 return -EINVAL;
4544         }
4545
4546         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
4547                       modifier == I915_FORMAT_MOD_Yf_TILED ||
4548                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4549                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4550         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
4551         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4552                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4553         wp->is_planar = drm_format_info_is_yuv_semiplanar(format);
4554
4555         wp->width = width;
4556         if (color_plane == 1 && wp->is_planar)
4557                 wp->width /= 2;
4558
4559         wp->cpp = format->cpp[color_plane];
4560         wp->plane_pixel_rate = plane_pixel_rate;
4561
4562         if (INTEL_GEN(dev_priv) >= 11 &&
4563             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
4564                 wp->dbuf_block_size = 256;
4565         else
4566                 wp->dbuf_block_size = 512;
4567
4568         if (drm_rotation_90_or_270(rotation)) {
4569                 switch (wp->cpp) {
4570                 case 1:
4571                         wp->y_min_scanlines = 16;
4572                         break;
4573                 case 2:
4574                         wp->y_min_scanlines = 8;
4575                         break;
4576                 case 4:
4577                         wp->y_min_scanlines = 4;
4578                         break;
4579                 default:
4580                         MISSING_CASE(wp->cpp);
4581                         return -EINVAL;
4582                 }
4583         } else {
4584                 wp->y_min_scanlines = 4;
4585         }
4586
4587         if (skl_needs_memory_bw_wa(dev_priv))
4588                 wp->y_min_scanlines *= 2;
4589
4590         wp->plane_bytes_per_line = wp->width * wp->cpp;
4591         if (wp->y_tiled) {
4592                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4593                                            wp->y_min_scanlines,
4594                                            wp->dbuf_block_size);
4595
4596                 if (INTEL_GEN(dev_priv) >= 10)
4597                         interm_pbpl++;
4598
4599                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4600                                                         wp->y_min_scanlines);
4601         } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) {
4602                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4603                                            wp->dbuf_block_size);
4604                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4605         } else {
4606                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4607                                            wp->dbuf_block_size) + 1;
4608                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4609         }
4610
4611         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4612                                              wp->plane_blocks_per_line);
4613
4614         wp->linetime_us = fixed16_to_u32_round_up(
4615                                         intel_get_linetime_us(crtc_state));
4616
4617         return 0;
4618 }
4619
4620 static int
4621 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
4622                             const struct intel_plane_state *plane_state,
4623                             struct skl_wm_params *wp, int color_plane)
4624 {
4625         const struct drm_framebuffer *fb = plane_state->base.fb;
4626         int width;
4627
4628         /*
4629          * Src coordinates are already rotated by 270 degrees for
4630          * the 90/270 degree plane rotation cases (to match the
4631          * GTT mapping), hence no need to account for rotation here.
4632          */
4633         width = drm_rect_width(&plane_state->base.src) >> 16;
4634
4635         return skl_compute_wm_params(crtc_state, width,
4636                                      fb->format, fb->modifier,
4637                                      plane_state->base.rotation,
4638                                      skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
4639                                      wp, color_plane);
4640 }
4641
4642 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
4643 {
4644         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4645                 return true;
4646
4647         /* The number of lines are ignored for the level 0 watermark. */
4648         return level > 0;
4649 }
4650
4651 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4652                                  int level,
4653                                  const struct skl_wm_params *wp,
4654                                  const struct skl_wm_level *result_prev,
4655                                  struct skl_wm_level *result /* out */)
4656 {
4657         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4658         u32 latency = dev_priv->wm.skl_latency[level];
4659         uint_fixed_16_16_t method1, method2;
4660         uint_fixed_16_16_t selected_result;
4661         u32 res_blocks, res_lines, min_ddb_alloc = 0;
4662
4663         if (latency == 0) {
4664                 /* reject it */
4665                 result->min_ddb_alloc = U16_MAX;
4666                 return;
4667         }
4668
4669         /*
4670          * WaIncreaseLatencyIPCEnabled: kbl,cfl
4671          * Display WA #1141: kbl,cfl
4672          */
4673         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) ||
4674             dev_priv->ipc_enabled)
4675                 latency += 4;
4676
4677         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
4678                 latency += 15;
4679
4680         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4681                                  wp->cpp, latency, wp->dbuf_block_size);
4682         method2 = skl_wm_method2(wp->plane_pixel_rate,
4683                                  crtc_state->base.adjusted_mode.crtc_htotal,
4684                                  latency,
4685                                  wp->plane_blocks_per_line);
4686
4687         if (wp->y_tiled) {
4688                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4689         } else {
4690                 if ((wp->cpp * crtc_state->base.adjusted_mode.crtc_htotal /
4691                      wp->dbuf_block_size < 1) &&
4692                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
4693                         selected_result = method2;
4694                 } else if (latency >= wp->linetime_us) {
4695                         if (IS_GEN(dev_priv, 9) &&
4696                             !IS_GEMINILAKE(dev_priv))
4697                                 selected_result = min_fixed16(method1, method2);
4698                         else
4699                                 selected_result = method2;
4700                 } else {
4701                         selected_result = method1;
4702                 }
4703         }
4704
4705         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4706         res_lines = div_round_up_fixed16(selected_result,
4707                                          wp->plane_blocks_per_line);
4708
4709         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
4710                 /* Display WA #1125: skl,bxt,kbl */
4711                 if (level == 0 && wp->rc_surface)
4712                         res_blocks +=
4713                                 fixed16_to_u32_round_up(wp->y_tile_minimum);
4714
4715                 /* Display WA #1126: skl,bxt,kbl */
4716                 if (level >= 1 && level <= 7) {
4717                         if (wp->y_tiled) {
4718                                 res_blocks +=
4719                                     fixed16_to_u32_round_up(wp->y_tile_minimum);
4720                                 res_lines += wp->y_min_scanlines;
4721                         } else {
4722                                 res_blocks++;
4723                         }
4724
4725                         /*
4726                          * Make sure result blocks for higher latency levels are
4727                          * atleast as high as level below the current level.
4728                          * Assumption in DDB algorithm optimization for special
4729                          * cases. Also covers Display WA #1125 for RC.
4730                          */
4731                         if (result_prev->plane_res_b > res_blocks)
4732                                 res_blocks = result_prev->plane_res_b;
4733                 }
4734         }
4735
4736         if (INTEL_GEN(dev_priv) >= 11) {
4737                 if (wp->y_tiled) {
4738                         int extra_lines;
4739
4740                         if (res_lines % wp->y_min_scanlines == 0)
4741                                 extra_lines = wp->y_min_scanlines;
4742                         else
4743                                 extra_lines = wp->y_min_scanlines * 2 -
4744                                         res_lines % wp->y_min_scanlines;
4745
4746                         min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
4747                                                                  wp->plane_blocks_per_line);
4748                 } else {
4749                         min_ddb_alloc = res_blocks +
4750                                 DIV_ROUND_UP(res_blocks, 10);
4751                 }
4752         }
4753
4754         if (!skl_wm_has_lines(dev_priv, level))
4755                 res_lines = 0;
4756
4757         if (res_lines > 31) {
4758                 /* reject it */
4759                 result->min_ddb_alloc = U16_MAX;
4760                 return;
4761         }
4762
4763         /*
4764          * If res_lines is valid, assume we can use this watermark level
4765          * for now.  We'll come back and disable it after we calculate the
4766          * DDB allocation if it turns out we don't actually have enough
4767          * blocks to satisfy it.
4768          */
4769         result->plane_res_b = res_blocks;
4770         result->plane_res_l = res_lines;
4771         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
4772         result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
4773         result->plane_en = true;
4774 }
4775
4776 static void
4777 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
4778                       const struct skl_wm_params *wm_params,
4779                       struct skl_wm_level *levels)
4780 {
4781         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4782         int level, max_level = ilk_wm_max_level(dev_priv);
4783         struct skl_wm_level *result_prev = &levels[0];
4784
4785         for (level = 0; level <= max_level; level++) {
4786                 struct skl_wm_level *result = &levels[level];
4787
4788                 skl_compute_plane_wm(crtc_state, level, wm_params,
4789                                      result_prev, result);
4790
4791                 result_prev = result;
4792         }
4793 }
4794
4795 static u32
4796 skl_compute_linetime_wm(const struct intel_crtc_state *crtc_state)
4797 {
4798         struct drm_atomic_state *state = crtc_state->base.state;
4799         struct drm_i915_private *dev_priv = to_i915(state->dev);
4800         uint_fixed_16_16_t linetime_us;
4801         u32 linetime_wm;
4802
4803         linetime_us = intel_get_linetime_us(crtc_state);
4804         linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4805
4806         /* Display WA #1135: BXT:ALL GLK:ALL */
4807         if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled)
4808                 linetime_wm /= 2;
4809
4810         return linetime_wm;
4811 }
4812
4813 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
4814                                       const struct skl_wm_params *wp,
4815                                       struct skl_plane_wm *wm)
4816 {
4817         struct drm_device *dev = crtc_state->base.crtc->dev;
4818         const struct drm_i915_private *dev_priv = to_i915(dev);
4819         u16 trans_min, trans_y_tile_min;
4820         const u16 trans_amount = 10; /* This is configurable amount */
4821         u16 wm0_sel_res_b, trans_offset_b, res_blocks;
4822
4823         /* Transition WM are not recommended by HW team for GEN9 */
4824         if (INTEL_GEN(dev_priv) <= 9)
4825                 return;
4826
4827         /* Transition WM don't make any sense if ipc is disabled */
4828         if (!dev_priv->ipc_enabled)
4829                 return;
4830
4831         trans_min = 14;
4832         if (INTEL_GEN(dev_priv) >= 11)
4833                 trans_min = 4;
4834
4835         trans_offset_b = trans_min + trans_amount;
4836
4837         /*
4838          * The spec asks for Selected Result Blocks for wm0 (the real value),
4839          * not Result Blocks (the integer value). Pay attention to the capital
4840          * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
4841          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
4842          * and since we later will have to get the ceiling of the sum in the
4843          * transition watermarks calculation, we can just pretend Selected
4844          * Result Blocks is Result Blocks minus 1 and it should work for the
4845          * current platforms.
4846          */
4847         wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
4848
4849         if (wp->y_tiled) {
4850                 trans_y_tile_min =
4851                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
4852                 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
4853                                 trans_offset_b;
4854         } else {
4855                 res_blocks = wm0_sel_res_b + trans_offset_b;
4856
4857                 /* WA BUG:1938466 add one block for non y-tile planes */
4858                 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4859                         res_blocks += 1;
4860
4861         }
4862
4863         /*
4864          * Just assume we can enable the transition watermark.  After
4865          * computing the DDB we'll come back and disable it if that
4866          * assumption turns out to be false.
4867          */
4868         wm->trans_wm.plane_res_b = res_blocks + 1;
4869         wm->trans_wm.plane_en = true;
4870 }
4871
4872 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
4873                                      const struct intel_plane_state *plane_state,
4874                                      enum plane_id plane_id, int color_plane)
4875 {
4876         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
4877         struct skl_wm_params wm_params;
4878         int ret;
4879
4880         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
4881                                           &wm_params, color_plane);
4882         if (ret)
4883                 return ret;
4884
4885         skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
4886         skl_compute_transition_wm(crtc_state, &wm_params, wm);
4887
4888         return 0;
4889 }
4890
4891 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
4892                                  const struct intel_plane_state *plane_state,
4893                                  enum plane_id plane_id)
4894 {
4895         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
4896         struct skl_wm_params wm_params;
4897         int ret;
4898
4899         wm->is_planar = true;
4900
4901         /* uv plane watermarks must also be validated for NV12/Planar */
4902         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
4903                                           &wm_params, 1);
4904         if (ret)
4905                 return ret;
4906
4907         skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
4908
4909         return 0;
4910 }
4911
4912 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
4913                               const struct intel_plane_state *plane_state)
4914 {
4915         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4916         const struct drm_framebuffer *fb = plane_state->base.fb;
4917         enum plane_id plane_id = plane->id;
4918         int ret;
4919
4920         if (!intel_wm_plane_visible(crtc_state, plane_state))
4921                 return 0;
4922
4923         ret = skl_build_plane_wm_single(crtc_state, plane_state,
4924                                         plane_id, 0);
4925         if (ret)
4926                 return ret;
4927
4928         if (fb->format->is_yuv && fb->format->num_planes > 1) {
4929                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
4930                                             plane_id);
4931                 if (ret)
4932                         return ret;
4933         }
4934
4935         return 0;
4936 }
4937
4938 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
4939                               const struct intel_plane_state *plane_state)
4940 {
4941         enum plane_id plane_id = to_intel_plane(plane_state->base.plane)->id;
4942         int ret;
4943
4944         /* Watermarks calculated in master */
4945         if (plane_state->planar_slave)
4946                 return 0;
4947
4948         if (plane_state->planar_linked_plane) {
4949                 const struct drm_framebuffer *fb = plane_state->base.fb;
4950                 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
4951
4952                 WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state));
4953                 WARN_ON(!fb->format->is_yuv ||
4954                         fb->format->num_planes == 1);
4955
4956                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
4957                                                 y_plane_id, 0);
4958                 if (ret)
4959                         return ret;
4960
4961                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
4962                                                 plane_id, 1);
4963                 if (ret)
4964                         return ret;
4965         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
4966                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
4967                                                 plane_id, 0);
4968                 if (ret)
4969                         return ret;
4970         }
4971
4972         return 0;
4973 }
4974
4975 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
4976 {
4977         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4978         struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4979         struct intel_plane *plane;
4980         const struct intel_plane_state *plane_state;
4981         int ret;
4982
4983         /*
4984          * We'll only calculate watermarks for planes that are actually
4985          * enabled, so make sure all other planes are set as disabled.
4986          */
4987         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4988
4989         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
4990                                                      crtc_state) {
4991
4992                 if (INTEL_GEN(dev_priv) >= 11)
4993                         ret = icl_build_plane_wm(crtc_state, plane_state);
4994                 else
4995                         ret = skl_build_plane_wm(crtc_state, plane_state);
4996                 if (ret)
4997                         return ret;
4998         }
4999
5000         pipe_wm->linetime = skl_compute_linetime_wm(crtc_state);
5001
5002         return 0;
5003 }
5004
5005 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5006                                 i915_reg_t reg,
5007                                 const struct skl_ddb_entry *entry)
5008 {
5009         if (entry->end)
5010                 I915_WRITE_FW(reg, (entry->end - 1) << 16 | entry->start);
5011         else
5012                 I915_WRITE_FW(reg, 0);
5013 }
5014
5015 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5016                                i915_reg_t reg,
5017                                const struct skl_wm_level *level)
5018 {
5019         u32 val = 0;
5020
5021         if (level->plane_en)
5022                 val |= PLANE_WM_EN;
5023         if (level->ignore_lines)
5024                 val |= PLANE_WM_IGNORE_LINES;
5025         val |= level->plane_res_b;
5026         val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5027
5028         I915_WRITE_FW(reg, val);
5029 }
5030
5031 void skl_write_plane_wm(struct intel_plane *plane,
5032                         const struct intel_crtc_state *crtc_state)
5033 {
5034         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5035         int level, max_level = ilk_wm_max_level(dev_priv);
5036         enum plane_id plane_id = plane->id;
5037         enum pipe pipe = plane->pipe;
5038         const struct skl_plane_wm *wm =
5039                 &crtc_state->wm.skl.optimal.planes[plane_id];
5040         const struct skl_ddb_entry *ddb_y =
5041                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5042         const struct skl_ddb_entry *ddb_uv =
5043                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5044
5045         for (level = 0; level <= max_level; level++) {
5046                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5047                                    &wm->wm[level]);
5048         }
5049         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5050                            &wm->trans_wm);
5051
5052         if (INTEL_GEN(dev_priv) >= 11) {
5053                 skl_ddb_entry_write(dev_priv,
5054                                     PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5055                 return;
5056         }
5057
5058         if (wm->is_planar)
5059                 swap(ddb_y, ddb_uv);
5060
5061         skl_ddb_entry_write(dev_priv,
5062                             PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5063         skl_ddb_entry_write(dev_priv,
5064                             PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5065 }
5066
5067 void skl_write_cursor_wm(struct intel_plane *plane,
5068                          const struct intel_crtc_state *crtc_state)
5069 {
5070         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5071         int level, max_level = ilk_wm_max_level(dev_priv);
5072         enum plane_id plane_id = plane->id;
5073         enum pipe pipe = plane->pipe;
5074         const struct skl_plane_wm *wm =
5075                 &crtc_state->wm.skl.optimal.planes[plane_id];
5076         const struct skl_ddb_entry *ddb =
5077                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5078
5079         for (level = 0; level <= max_level; level++) {
5080                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5081                                    &wm->wm[level]);
5082         }
5083         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5084
5085         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5086 }
5087
5088 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5089                          const struct skl_wm_level *l2)
5090 {
5091         return l1->plane_en == l2->plane_en &&
5092                 l1->ignore_lines == l2->ignore_lines &&
5093                 l1->plane_res_l == l2->plane_res_l &&
5094                 l1->plane_res_b == l2->plane_res_b;
5095 }
5096
5097 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5098                                 const struct skl_plane_wm *wm1,
5099                                 const struct skl_plane_wm *wm2)
5100 {
5101         int level, max_level = ilk_wm_max_level(dev_priv);
5102
5103         for (level = 0; level <= max_level; level++) {
5104                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) ||
5105                     !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level]))
5106                         return false;
5107         }
5108
5109         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5110 }
5111
5112 static bool skl_pipe_wm_equals(struct intel_crtc *crtc,
5113                                const struct skl_pipe_wm *wm1,
5114                                const struct skl_pipe_wm *wm2)
5115 {
5116         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5117         enum plane_id plane_id;
5118
5119         for_each_plane_id_on_crtc(crtc, plane_id) {
5120                 if (!skl_plane_wm_equals(dev_priv,
5121                                          &wm1->planes[plane_id],
5122                                          &wm2->planes[plane_id]))
5123                         return false;
5124         }
5125
5126         return wm1->linetime == wm2->linetime;
5127 }
5128
5129 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5130                                            const struct skl_ddb_entry *b)
5131 {
5132         return a->start < b->end && b->start < a->end;
5133 }
5134
5135 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5136                                  const struct skl_ddb_entry *entries,
5137                                  int num_entries, int ignore_idx)
5138 {
5139         int i;
5140
5141         for (i = 0; i < num_entries; i++) {
5142                 if (i != ignore_idx &&
5143                     skl_ddb_entries_overlap(ddb, &entries[i]))
5144                         return true;
5145         }
5146
5147         return false;
5148 }
5149
5150 static int
5151 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5152                             struct intel_crtc_state *new_crtc_state)
5153 {
5154         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->base.state);
5155         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5156         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5157         struct intel_plane *plane;
5158
5159         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5160                 struct intel_plane_state *plane_state;
5161                 enum plane_id plane_id = plane->id;
5162
5163                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5164                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5165                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5166                                         &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5167                         continue;
5168
5169                 plane_state = intel_atomic_get_plane_state(state, plane);
5170                 if (IS_ERR(plane_state))
5171                         return PTR_ERR(plane_state);
5172
5173                 new_crtc_state->update_planes |= BIT(plane_id);
5174         }
5175
5176         return 0;
5177 }
5178
5179 static int
5180 skl_compute_ddb(struct intel_atomic_state *state)
5181 {
5182         const struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5183         struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5184         struct intel_crtc_state *old_crtc_state;
5185         struct intel_crtc_state *new_crtc_state;
5186         struct intel_crtc *crtc;
5187         int ret, i;
5188
5189         memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5190
5191         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5192                                             new_crtc_state, i) {
5193                 ret = skl_allocate_pipe_ddb(new_crtc_state, ddb);
5194                 if (ret)
5195                         return ret;
5196
5197                 ret = skl_ddb_add_affected_planes(old_crtc_state,
5198                                                   new_crtc_state);
5199                 if (ret)
5200                         return ret;
5201         }
5202
5203         return 0;
5204 }
5205
5206 static char enast(bool enable)
5207 {
5208         return enable ? '*' : ' ';
5209 }
5210
5211 static void
5212 skl_print_wm_changes(struct intel_atomic_state *state)
5213 {
5214         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5215         const struct intel_crtc_state *old_crtc_state;
5216         const struct intel_crtc_state *new_crtc_state;
5217         struct intel_plane *plane;
5218         struct intel_crtc *crtc;
5219         int i;
5220
5221         if ((drm_debug & DRM_UT_KMS) == 0)
5222                 return;
5223
5224         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5225                                             new_crtc_state, i) {
5226                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5227
5228                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5229                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5230
5231                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5232                         enum plane_id plane_id = plane->id;
5233                         const struct skl_ddb_entry *old, *new;
5234
5235                         old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5236                         new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5237
5238                         if (skl_ddb_entry_equal(old, new))
5239                                 continue;
5240
5241                         DRM_DEBUG_KMS("[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5242                                       plane->base.base.id, plane->base.name,
5243                                       old->start, old->end, new->start, new->end,
5244                                       skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5245                 }
5246
5247                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5248                         enum plane_id plane_id = plane->id;
5249                         const struct skl_plane_wm *old_wm, *new_wm;
5250
5251                         old_wm = &old_pipe_wm->planes[plane_id];
5252                         new_wm = &new_pipe_wm->planes[plane_id];
5253
5254                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5255                                 continue;
5256
5257                         DRM_DEBUG_KMS("[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm"
5258                                       " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm\n",
5259                                       plane->base.base.id, plane->base.name,
5260                                       enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5261                                       enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5262                                       enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5263                                       enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5264                                       enast(old_wm->trans_wm.plane_en),
5265                                       enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5266                                       enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5267                                       enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5268                                       enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5269                                       enast(new_wm->trans_wm.plane_en));
5270
5271                         DRM_DEBUG_KMS("[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5272                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5273                                       plane->base.base.id, plane->base.name,
5274                                       enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5275                                       enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5276                                       enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5277                                       enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5278                                       enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5279                                       enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5280                                       enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5281                                       enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5282                                       enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5283
5284                                       enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5285                                       enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5286                                       enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5287                                       enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5288                                       enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5289                                       enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5290                                       enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5291                                       enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5292                                       enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l);
5293
5294                         DRM_DEBUG_KMS("[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5295                                       " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5296                                       plane->base.base.id, plane->base.name,
5297                                       old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5298                                       old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5299                                       old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5300                                       old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5301                                       old_wm->trans_wm.plane_res_b,
5302                                       new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5303                                       new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5304                                       new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5305                                       new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5306                                       new_wm->trans_wm.plane_res_b);
5307
5308                         DRM_DEBUG_KMS("[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5309                                       " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5310                                       plane->base.base.id, plane->base.name,
5311                                       old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5312                                       old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5313                                       old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5314                                       old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5315                                       old_wm->trans_wm.min_ddb_alloc,
5316                                       new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5317                                       new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5318                                       new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5319                                       new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5320                                       new_wm->trans_wm.min_ddb_alloc);
5321                 }
5322         }
5323 }
5324
5325 static int intel_add_all_pipes(struct intel_atomic_state *state)
5326 {
5327         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5328         struct intel_crtc *crtc;
5329
5330         for_each_intel_crtc(&dev_priv->drm, crtc) {
5331                 struct intel_crtc_state *crtc_state;
5332
5333                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5334                 if (IS_ERR(crtc_state))
5335                         return PTR_ERR(crtc_state);
5336         }
5337
5338         return 0;
5339 }
5340
5341 static int
5342 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5343 {
5344         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5345         int ret;
5346
5347         /*
5348          * If this is our first atomic update following hardware readout,
5349          * we can't trust the DDB that the BIOS programmed for us.  Let's
5350          * pretend that all pipes switched active status so that we'll
5351          * ensure a full DDB recompute.
5352          */
5353         if (dev_priv->wm.distrust_bios_wm) {
5354                 ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
5355                                        state->base.acquire_ctx);
5356                 if (ret)
5357                         return ret;
5358
5359                 state->active_pipe_changes = ~0;
5360
5361                 /*
5362                  * We usually only initialize state->active_pipes if we
5363                  * we're doing a modeset; make sure this field is always
5364                  * initialized during the sanitization process that happens
5365                  * on the first commit too.
5366                  */
5367                 if (!state->modeset)
5368                         state->active_pipes = dev_priv->active_pipes;
5369         }
5370
5371         /*
5372          * If the modeset changes which CRTC's are active, we need to
5373          * recompute the DDB allocation for *all* active pipes, even
5374          * those that weren't otherwise being modified in any way by this
5375          * atomic commit.  Due to the shrinking of the per-pipe allocations
5376          * when new active CRTC's are added, it's possible for a pipe that
5377          * we were already using and aren't changing at all here to suddenly
5378          * become invalid if its DDB needs exceeds its new allocation.
5379          *
5380          * Note that if we wind up doing a full DDB recompute, we can't let
5381          * any other display updates race with this transaction, so we need
5382          * to grab the lock on *all* CRTC's.
5383          */
5384         if (state->active_pipe_changes || state->modeset) {
5385                 state->wm_results.dirty_pipes = ~0;
5386
5387                 ret = intel_add_all_pipes(state);
5388                 if (ret)
5389                         return ret;
5390         }
5391
5392         return 0;
5393 }
5394
5395 /*
5396  * To make sure the cursor watermark registers are always consistent
5397  * with our computed state the following scenario needs special
5398  * treatment:
5399  *
5400  * 1. enable cursor
5401  * 2. move cursor entirely offscreen
5402  * 3. disable cursor
5403  *
5404  * Step 2. does call .disable_plane() but does not zero the watermarks
5405  * (since we consider an offscreen cursor still active for the purposes
5406  * of watermarks). Step 3. would not normally call .disable_plane()
5407  * because the actual plane visibility isn't changing, and we don't
5408  * deallocate the cursor ddb until the pipe gets disabled. So we must
5409  * force step 3. to call .disable_plane() to update the watermark
5410  * registers properly.
5411  *
5412  * Other planes do not suffer from this issues as their watermarks are
5413  * calculated based on the actual plane visibility. The only time this
5414  * can trigger for the other planes is during the initial readout as the
5415  * default value of the watermarks registers is not zero.
5416  */
5417 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
5418                                       struct intel_crtc *crtc)
5419 {
5420         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5421         const struct intel_crtc_state *old_crtc_state =
5422                 intel_atomic_get_old_crtc_state(state, crtc);
5423         struct intel_crtc_state *new_crtc_state =
5424                 intel_atomic_get_new_crtc_state(state, crtc);
5425         struct intel_plane *plane;
5426
5427         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5428                 struct intel_plane_state *plane_state;
5429                 enum plane_id plane_id = plane->id;
5430
5431                 /*
5432                  * Force a full wm update for every plane on modeset.
5433                  * Required because the reset value of the wm registers
5434                  * is non-zero, whereas we want all disabled planes to
5435                  * have zero watermarks. So if we turn off the relevant
5436                  * power well the hardware state will go out of sync
5437                  * with the software state.
5438                  */
5439                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->base) &&
5440                     skl_plane_wm_equals(dev_priv,
5441                                         &old_crtc_state->wm.skl.optimal.planes[plane_id],
5442                                         &new_crtc_state->wm.skl.optimal.planes[plane_id]))
5443                         continue;
5444
5445                 plane_state = intel_atomic_get_plane_state(state, plane);
5446                 if (IS_ERR(plane_state))
5447                         return PTR_ERR(plane_state);
5448
5449                 new_crtc_state->update_planes |= BIT(plane_id);
5450         }
5451
5452         return 0;
5453 }
5454
5455 static int
5456 skl_compute_wm(struct intel_atomic_state *state)
5457 {
5458         struct intel_crtc *crtc;
5459         struct intel_crtc_state *new_crtc_state;
5460         struct intel_crtc_state *old_crtc_state;
5461         struct skl_ddb_values *results = &state->wm_results;
5462         int ret, i;
5463
5464         /* Clear all dirty flags */
5465         results->dirty_pipes = 0;
5466
5467         ret = skl_ddb_add_affected_pipes(state);
5468         if (ret)
5469                 return ret;
5470
5471         /*
5472          * Calculate WM's for all pipes that are part of this transaction.
5473          * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
5474          * weren't otherwise being modified (and set bits in dirty_pipes) if
5475          * pipe allocations had to change.
5476          */
5477         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5478                                             new_crtc_state, i) {
5479                 ret = skl_build_pipe_wm(new_crtc_state);
5480                 if (ret)
5481                         return ret;
5482
5483                 ret = skl_wm_add_affected_planes(state, crtc);
5484                 if (ret)
5485                         return ret;
5486
5487                 if (!skl_pipe_wm_equals(crtc,
5488                                         &old_crtc_state->wm.skl.optimal,
5489                                         &new_crtc_state->wm.skl.optimal))
5490                         results->dirty_pipes |= BIT(crtc->pipe);
5491         }
5492
5493         ret = skl_compute_ddb(state);
5494         if (ret)
5495                 return ret;
5496
5497         skl_print_wm_changes(state);
5498
5499         return 0;
5500 }
5501
5502 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5503                                       struct intel_crtc_state *crtc_state)
5504 {
5505         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5506         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5507         struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5508         enum pipe pipe = crtc->pipe;
5509
5510         if ((state->wm_results.dirty_pipes & BIT(crtc->pipe)) == 0)
5511                 return;
5512
5513         I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5514 }
5515
5516 static void skl_initial_wm(struct intel_atomic_state *state,
5517                            struct intel_crtc_state *crtc_state)
5518 {
5519         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5520         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5521         struct skl_ddb_values *results = &state->wm_results;
5522
5523         if ((results->dirty_pipes & BIT(crtc->pipe)) == 0)
5524                 return;
5525
5526         mutex_lock(&dev_priv->wm.wm_mutex);
5527
5528         if (crtc_state->base.active_changed)
5529                 skl_atomic_update_crtc_wm(state, crtc_state);
5530
5531         mutex_unlock(&dev_priv->wm.wm_mutex);
5532 }
5533
5534 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
5535                                   struct intel_wm_config *config)
5536 {
5537         struct intel_crtc *crtc;
5538
5539         /* Compute the currently _active_ config */
5540         for_each_intel_crtc(&dev_priv->drm, crtc) {
5541                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5542
5543                 if (!wm->pipe_enabled)
5544                         continue;
5545
5546                 config->sprites_enabled |= wm->sprites_enabled;
5547                 config->sprites_scaled |= wm->sprites_scaled;
5548                 config->num_pipes_active++;
5549         }
5550 }
5551
5552 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5553 {
5554         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5555         struct ilk_wm_maximums max;
5556         struct intel_wm_config config = {};
5557         struct ilk_wm_values results = {};
5558         enum intel_ddb_partitioning partitioning;
5559
5560         ilk_compute_wm_config(dev_priv, &config);
5561
5562         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
5563         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
5564
5565         /* 5/6 split only in single pipe config on IVB+ */
5566         if (INTEL_GEN(dev_priv) >= 7 &&
5567             config.num_pipes_active == 1 && config.sprites_enabled) {
5568                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
5569                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
5570
5571                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
5572         } else {
5573                 best_lp_wm = &lp_wm_1_2;
5574         }
5575
5576         partitioning = (best_lp_wm == &lp_wm_1_2) ?
5577                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5578
5579         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
5580
5581         ilk_write_wm_values(dev_priv, &results);
5582 }
5583
5584 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5585                                    struct intel_crtc_state *crtc_state)
5586 {
5587         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
5588         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5589
5590         mutex_lock(&dev_priv->wm.wm_mutex);
5591         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
5592         ilk_program_watermarks(dev_priv);
5593         mutex_unlock(&dev_priv->wm.wm_mutex);
5594 }
5595
5596 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5597                                     struct intel_crtc_state *crtc_state)
5598 {
5599         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
5600         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5601
5602         if (!crtc_state->wm.need_postvbl_update)
5603                 return;
5604
5605         mutex_lock(&dev_priv->wm.wm_mutex);
5606         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
5607         ilk_program_watermarks(dev_priv);
5608         mutex_unlock(&dev_priv->wm.wm_mutex);
5609 }
5610
5611 static inline void skl_wm_level_from_reg_val(u32 val,
5612                                              struct skl_wm_level *level)
5613 {
5614         level->plane_en = val & PLANE_WM_EN;
5615         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
5616         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5617         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5618                 PLANE_WM_LINES_MASK;
5619 }
5620
5621 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
5622                               struct skl_pipe_wm *out)
5623 {
5624         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5625         enum pipe pipe = crtc->pipe;
5626         int level, max_level;
5627         enum plane_id plane_id;
5628         u32 val;
5629
5630         max_level = ilk_wm_max_level(dev_priv);
5631
5632         for_each_plane_id_on_crtc(crtc, plane_id) {
5633                 struct skl_plane_wm *wm = &out->planes[plane_id];
5634
5635                 for (level = 0; level <= max_level; level++) {
5636                         if (plane_id != PLANE_CURSOR)
5637                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5638                         else
5639                                 val = I915_READ(CUR_WM(pipe, level));
5640
5641                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
5642                 }
5643
5644                 if (plane_id != PLANE_CURSOR)
5645                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5646                 else
5647                         val = I915_READ(CUR_WM_TRANS(pipe));
5648
5649                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5650         }
5651
5652         if (!crtc->active)
5653                 return;
5654
5655         out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5656 }
5657
5658 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
5659 {
5660         struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5661         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5662         struct intel_crtc *crtc;
5663         struct intel_crtc_state *crtc_state;
5664
5665         skl_ddb_get_hw_state(dev_priv, ddb);
5666         for_each_intel_crtc(&dev_priv->drm, crtc) {
5667                 crtc_state = to_intel_crtc_state(crtc->base.state);
5668
5669                 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
5670
5671                 if (crtc->active)
5672                         hw->dirty_pipes |= BIT(crtc->pipe);
5673         }
5674
5675         if (dev_priv->active_pipes) {
5676                 /* Fully recompute DDB on first atomic commit */
5677                 dev_priv->wm.distrust_bios_wm = true;
5678         }
5679 }
5680
5681 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
5682 {
5683         struct drm_device *dev = crtc->base.dev;
5684         struct drm_i915_private *dev_priv = to_i915(dev);
5685         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5686         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
5687         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
5688         enum pipe pipe = crtc->pipe;
5689         static const i915_reg_t wm0_pipe_reg[] = {
5690                 [PIPE_A] = WM0_PIPEA_ILK,
5691                 [PIPE_B] = WM0_PIPEB_ILK,
5692                 [PIPE_C] = WM0_PIPEC_IVB,
5693         };
5694
5695         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5696         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5697                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5698
5699         memset(active, 0, sizeof(*active));
5700
5701         active->pipe_enabled = crtc->active;
5702
5703         if (active->pipe_enabled) {
5704                 u32 tmp = hw->wm_pipe[pipe];
5705
5706                 /*
5707                  * For active pipes LP0 watermark is marked as
5708                  * enabled, and LP1+ watermaks as disabled since
5709                  * we can't really reverse compute them in case
5710                  * multiple pipes are active.
5711                  */
5712                 active->wm[0].enable = true;
5713                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5714                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5715                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5716                 active->linetime = hw->wm_linetime[pipe];
5717         } else {
5718                 int level, max_level = ilk_wm_max_level(dev_priv);
5719
5720                 /*
5721                  * For inactive pipes, all watermark levels
5722                  * should be marked as enabled but zeroed,
5723                  * which is what we'd compute them to.
5724                  */
5725                 for (level = 0; level <= max_level; level++)
5726                         active->wm[level].enable = true;
5727         }
5728
5729         crtc->wm.active.ilk = *active;
5730 }
5731
5732 #define _FW_WM(value, plane) \
5733         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5734 #define _FW_WM_VLV(value, plane) \
5735         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5736
5737 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5738                                struct g4x_wm_values *wm)
5739 {
5740         u32 tmp;
5741
5742         tmp = I915_READ(DSPFW1);
5743         wm->sr.plane = _FW_WM(tmp, SR);
5744         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5745         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5746         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5747
5748         tmp = I915_READ(DSPFW2);
5749         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5750         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5751         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5752         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5753         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5754         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5755
5756         tmp = I915_READ(DSPFW3);
5757         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5758         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5759         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5760         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5761 }
5762
5763 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5764                                struct vlv_wm_values *wm)
5765 {
5766         enum pipe pipe;
5767         u32 tmp;
5768
5769         for_each_pipe(dev_priv, pipe) {
5770                 tmp = I915_READ(VLV_DDL(pipe));
5771
5772                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5773                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5774                 wm->ddl[pipe].plane[PLANE_CURSOR] =
5775                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5776                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5777                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5778                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5779                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5780         }
5781
5782         tmp = I915_READ(DSPFW1);
5783         wm->sr.plane = _FW_WM(tmp, SR);
5784         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5785         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5786         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5787
5788         tmp = I915_READ(DSPFW2);
5789         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5790         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5791         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5792
5793         tmp = I915_READ(DSPFW3);
5794         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5795
5796         if (IS_CHERRYVIEW(dev_priv)) {
5797                 tmp = I915_READ(DSPFW7_CHV);
5798                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5799                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5800
5801                 tmp = I915_READ(DSPFW8_CHV);
5802                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5803                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5804
5805                 tmp = I915_READ(DSPFW9_CHV);
5806                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5807                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5808
5809                 tmp = I915_READ(DSPHOWM);
5810                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5811                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5812                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5813                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5814                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5815                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5816                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5817                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5818                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5819                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5820         } else {
5821                 tmp = I915_READ(DSPFW7);
5822                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5823                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5824
5825                 tmp = I915_READ(DSPHOWM);
5826                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5827                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5828                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5829                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5830                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5831                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5832                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5833         }
5834 }
5835
5836 #undef _FW_WM
5837 #undef _FW_WM_VLV
5838
5839 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
5840 {
5841         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5842         struct intel_crtc *crtc;
5843
5844         g4x_read_wm_values(dev_priv, wm);
5845
5846         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5847
5848         for_each_intel_crtc(&dev_priv->drm, crtc) {
5849                 struct intel_crtc_state *crtc_state =
5850                         to_intel_crtc_state(crtc->base.state);
5851                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5852                 struct g4x_pipe_wm *raw;
5853                 enum pipe pipe = crtc->pipe;
5854                 enum plane_id plane_id;
5855                 int level, max_level;
5856
5857                 active->cxsr = wm->cxsr;
5858                 active->hpll_en = wm->hpll_en;
5859                 active->fbc_en = wm->fbc_en;
5860
5861                 active->sr = wm->sr;
5862                 active->hpll = wm->hpll;
5863
5864                 for_each_plane_id_on_crtc(crtc, plane_id) {
5865                         active->wm.plane[plane_id] =
5866                                 wm->pipe[pipe].plane[plane_id];
5867                 }
5868
5869                 if (wm->cxsr && wm->hpll_en)
5870                         max_level = G4X_WM_LEVEL_HPLL;
5871                 else if (wm->cxsr)
5872                         max_level = G4X_WM_LEVEL_SR;
5873                 else
5874                         max_level = G4X_WM_LEVEL_NORMAL;
5875
5876                 level = G4X_WM_LEVEL_NORMAL;
5877                 raw = &crtc_state->wm.g4x.raw[level];
5878                 for_each_plane_id_on_crtc(crtc, plane_id)
5879                         raw->plane[plane_id] = active->wm.plane[plane_id];
5880
5881                 if (++level > max_level)
5882                         goto out;
5883
5884                 raw = &crtc_state->wm.g4x.raw[level];
5885                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
5886                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
5887                 raw->plane[PLANE_SPRITE0] = 0;
5888                 raw->fbc = active->sr.fbc;
5889
5890                 if (++level > max_level)
5891                         goto out;
5892
5893                 raw = &crtc_state->wm.g4x.raw[level];
5894                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
5895                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
5896                 raw->plane[PLANE_SPRITE0] = 0;
5897                 raw->fbc = active->hpll.fbc;
5898
5899         out:
5900                 for_each_plane_id_on_crtc(crtc, plane_id)
5901                         g4x_raw_plane_wm_set(crtc_state, level,
5902                                              plane_id, USHRT_MAX);
5903                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
5904
5905                 crtc_state->wm.g4x.optimal = *active;
5906                 crtc_state->wm.g4x.intermediate = *active;
5907
5908                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
5909                               pipe_name(pipe),
5910                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5911                               wm->pipe[pipe].plane[PLANE_CURSOR],
5912                               wm->pipe[pipe].plane[PLANE_SPRITE0]);
5913         }
5914
5915         DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
5916                       wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
5917         DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
5918                       wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
5919         DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
5920                       yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
5921 }
5922
5923 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
5924 {
5925         struct intel_plane *plane;
5926         struct intel_crtc *crtc;
5927
5928         mutex_lock(&dev_priv->wm.wm_mutex);
5929
5930         for_each_intel_plane(&dev_priv->drm, plane) {
5931                 struct intel_crtc *crtc =
5932                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5933                 struct intel_crtc_state *crtc_state =
5934                         to_intel_crtc_state(crtc->base.state);
5935                 struct intel_plane_state *plane_state =
5936                         to_intel_plane_state(plane->base.state);
5937                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
5938                 enum plane_id plane_id = plane->id;
5939                 int level;
5940
5941                 if (plane_state->base.visible)
5942                         continue;
5943
5944                 for (level = 0; level < 3; level++) {
5945                         struct g4x_pipe_wm *raw =
5946                                 &crtc_state->wm.g4x.raw[level];
5947
5948                         raw->plane[plane_id] = 0;
5949                         wm_state->wm.plane[plane_id] = 0;
5950                 }
5951
5952                 if (plane_id == PLANE_PRIMARY) {
5953                         for (level = 0; level < 3; level++) {
5954                                 struct g4x_pipe_wm *raw =
5955                                         &crtc_state->wm.g4x.raw[level];
5956                                 raw->fbc = 0;
5957                         }
5958
5959                         wm_state->sr.fbc = 0;
5960                         wm_state->hpll.fbc = 0;
5961                         wm_state->fbc_en = false;
5962                 }
5963         }
5964
5965         for_each_intel_crtc(&dev_priv->drm, crtc) {
5966                 struct intel_crtc_state *crtc_state =
5967                         to_intel_crtc_state(crtc->base.state);
5968
5969                 crtc_state->wm.g4x.intermediate =
5970                         crtc_state->wm.g4x.optimal;
5971                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
5972         }
5973
5974         g4x_program_watermarks(dev_priv);
5975
5976         mutex_unlock(&dev_priv->wm.wm_mutex);
5977 }
5978
5979 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
5980 {
5981         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
5982         struct intel_crtc *crtc;
5983         u32 val;
5984
5985         vlv_read_wm_values(dev_priv, wm);
5986
5987         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
5988         wm->level = VLV_WM_LEVEL_PM2;
5989
5990         if (IS_CHERRYVIEW(dev_priv)) {
5991                 vlv_punit_get(dev_priv);
5992
5993                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
5994                 if (val & DSP_MAXFIFO_PM5_ENABLE)
5995                         wm->level = VLV_WM_LEVEL_PM5;
5996
5997                 /*
5998                  * If DDR DVFS is disabled in the BIOS, Punit
5999                  * will never ack the request. So if that happens
6000                  * assume we don't have to enable/disable DDR DVFS
6001                  * dynamically. To test that just set the REQ_ACK
6002                  * bit to poke the Punit, but don't change the
6003                  * HIGH/LOW bits so that we don't actually change
6004                  * the current state.
6005                  */
6006                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6007                 val |= FORCE_DDR_FREQ_REQ_ACK;
6008                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6009
6010                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6011                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6012                         DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
6013                                       "assuming DDR DVFS is disabled\n");
6014                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6015                 } else {
6016                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6017                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6018                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6019                 }
6020
6021                 vlv_punit_put(dev_priv);
6022         }
6023
6024         for_each_intel_crtc(&dev_priv->drm, crtc) {
6025                 struct intel_crtc_state *crtc_state =
6026                         to_intel_crtc_state(crtc->base.state);
6027                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6028                 const struct vlv_fifo_state *fifo_state =
6029                         &crtc_state->wm.vlv.fifo_state;
6030                 enum pipe pipe = crtc->pipe;
6031                 enum plane_id plane_id;
6032                 int level;
6033
6034                 vlv_get_fifo_size(crtc_state);
6035
6036                 active->num_levels = wm->level + 1;
6037                 active->cxsr = wm->cxsr;
6038
6039                 for (level = 0; level < active->num_levels; level++) {
6040                         struct g4x_pipe_wm *raw =
6041                                 &crtc_state->wm.vlv.raw[level];
6042
6043                         active->sr[level].plane = wm->sr.plane;
6044                         active->sr[level].cursor = wm->sr.cursor;
6045
6046                         for_each_plane_id_on_crtc(crtc, plane_id) {
6047                                 active->wm[level].plane[plane_id] =
6048                                         wm->pipe[pipe].plane[plane_id];
6049
6050                                 raw->plane[plane_id] =
6051                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
6052                                                             fifo_state->plane[plane_id]);
6053                         }
6054                 }
6055
6056                 for_each_plane_id_on_crtc(crtc, plane_id)
6057                         vlv_raw_plane_wm_set(crtc_state, level,
6058                                              plane_id, USHRT_MAX);
6059                 vlv_invalidate_wms(crtc, active, level);
6060
6061                 crtc_state->wm.vlv.optimal = *active;
6062                 crtc_state->wm.vlv.intermediate = *active;
6063
6064                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6065                               pipe_name(pipe),
6066                               wm->pipe[pipe].plane[PLANE_PRIMARY],
6067                               wm->pipe[pipe].plane[PLANE_CURSOR],
6068                               wm->pipe[pipe].plane[PLANE_SPRITE0],
6069                               wm->pipe[pipe].plane[PLANE_SPRITE1]);
6070         }
6071
6072         DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6073                       wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6074 }
6075
6076 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6077 {
6078         struct intel_plane *plane;
6079         struct intel_crtc *crtc;
6080
6081         mutex_lock(&dev_priv->wm.wm_mutex);
6082
6083         for_each_intel_plane(&dev_priv->drm, plane) {
6084                 struct intel_crtc *crtc =
6085                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6086                 struct intel_crtc_state *crtc_state =
6087                         to_intel_crtc_state(crtc->base.state);
6088                 struct intel_plane_state *plane_state =
6089                         to_intel_plane_state(plane->base.state);
6090                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6091                 const struct vlv_fifo_state *fifo_state =
6092                         &crtc_state->wm.vlv.fifo_state;
6093                 enum plane_id plane_id = plane->id;
6094                 int level;
6095
6096                 if (plane_state->base.visible)
6097                         continue;
6098
6099                 for (level = 0; level < wm_state->num_levels; level++) {
6100                         struct g4x_pipe_wm *raw =
6101                                 &crtc_state->wm.vlv.raw[level];
6102
6103                         raw->plane[plane_id] = 0;
6104
6105                         wm_state->wm[level].plane[plane_id] =
6106                                 vlv_invert_wm_value(raw->plane[plane_id],
6107                                                     fifo_state->plane[plane_id]);
6108                 }
6109         }
6110
6111         for_each_intel_crtc(&dev_priv->drm, crtc) {
6112                 struct intel_crtc_state *crtc_state =
6113                         to_intel_crtc_state(crtc->base.state);
6114
6115                 crtc_state->wm.vlv.intermediate =
6116                         crtc_state->wm.vlv.optimal;
6117                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6118         }
6119
6120         vlv_program_watermarks(dev_priv);
6121
6122         mutex_unlock(&dev_priv->wm.wm_mutex);
6123 }
6124
6125 /*
6126  * FIXME should probably kill this and improve
6127  * the real watermark readout/sanitation instead
6128  */
6129 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6130 {
6131         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6132         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6133         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6134
6135         /*
6136          * Don't touch WM1S_LP_EN here.
6137          * Doing so could cause underruns.
6138          */
6139 }
6140
6141 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6142 {
6143         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6144         struct intel_crtc *crtc;
6145
6146         ilk_init_lp_watermarks(dev_priv);
6147
6148         for_each_intel_crtc(&dev_priv->drm, crtc)
6149                 ilk_pipe_wm_get_hw_state(crtc);
6150
6151         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6152         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6153         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6154
6155         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6156         if (INTEL_GEN(dev_priv) >= 7) {
6157                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6158                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6159         }
6160
6161         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6162                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6163                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6164         else if (IS_IVYBRIDGE(dev_priv))
6165                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6166                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6167
6168         hw->enable_fbc_wm =
6169                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6170 }
6171
6172 /**
6173  * intel_update_watermarks - update FIFO watermark values based on current modes
6174  * @crtc: the #intel_crtc on which to compute the WM
6175  *
6176  * Calculate watermark values for the various WM regs based on current mode
6177  * and plane configuration.
6178  *
6179  * There are several cases to deal with here:
6180  *   - normal (i.e. non-self-refresh)
6181  *   - self-refresh (SR) mode
6182  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6183  *   - lines are small relative to FIFO size (buffer can hold more than 2
6184  *     lines), so need to account for TLB latency
6185  *
6186  *   The normal calculation is:
6187  *     watermark = dotclock * bytes per pixel * latency
6188  *   where latency is platform & configuration dependent (we assume pessimal
6189  *   values here).
6190  *
6191  *   The SR calculation is:
6192  *     watermark = (trunc(latency/line time)+1) * surface width *
6193  *       bytes per pixel
6194  *   where
6195  *     line time = htotal / dotclock
6196  *     surface width = hdisplay for normal plane and 64 for cursor
6197  *   and latency is assumed to be high, as above.
6198  *
6199  * The final value programmed to the register should always be rounded up,
6200  * and include an extra 2 entries to account for clock crossings.
6201  *
6202  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6203  * to set the non-SR watermarks to 8.
6204  */
6205 void intel_update_watermarks(struct intel_crtc *crtc)
6206 {
6207         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6208
6209         if (dev_priv->display.update_wm)
6210                 dev_priv->display.update_wm(crtc);
6211 }
6212
6213 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6214 {
6215         u32 val;
6216
6217         if (!HAS_IPC(dev_priv))
6218                 return;
6219
6220         val = I915_READ(DISP_ARB_CTL2);
6221
6222         if (dev_priv->ipc_enabled)
6223                 val |= DISP_IPC_ENABLE;
6224         else
6225                 val &= ~DISP_IPC_ENABLE;
6226
6227         I915_WRITE(DISP_ARB_CTL2, val);
6228 }
6229
6230 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6231 {
6232         /* Display WA #0477 WaDisableIPC: skl */
6233         if (IS_SKYLAKE(dev_priv))
6234                 return false;
6235
6236         /* Display WA #1141: SKL:all KBL:all CFL */
6237         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
6238                 return dev_priv->dram_info.symmetric_memory;
6239
6240         return true;
6241 }
6242
6243 void intel_init_ipc(struct drm_i915_private *dev_priv)
6244 {
6245         if (!HAS_IPC(dev_priv))
6246                 return;
6247
6248         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6249
6250         intel_enable_ipc(dev_priv);
6251 }
6252
6253 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6254 {
6255         /*
6256          * On Ibex Peak and Cougar Point, we need to disable clock
6257          * gating for the panel power sequencer or it will fail to
6258          * start up when no ports are active.
6259          */
6260         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6261 }
6262
6263 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6264 {
6265         enum pipe pipe;
6266
6267         for_each_pipe(dev_priv, pipe) {
6268                 I915_WRITE(DSPCNTR(pipe),
6269                            I915_READ(DSPCNTR(pipe)) |
6270                            DISPPLANE_TRICKLE_FEED_DISABLE);
6271
6272                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6273                 POSTING_READ(DSPSURF(pipe));
6274         }
6275 }
6276
6277 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6278 {
6279         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6280
6281         /*
6282          * Required for FBC
6283          * WaFbcDisableDpfcClockGating:ilk
6284          */
6285         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6286                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6287                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6288
6289         I915_WRITE(PCH_3DCGDIS0,
6290                    MARIUNIT_CLOCK_GATE_DISABLE |
6291                    SVSMUNIT_CLOCK_GATE_DISABLE);
6292         I915_WRITE(PCH_3DCGDIS1,
6293                    VFMUNIT_CLOCK_GATE_DISABLE);
6294
6295         /*
6296          * According to the spec the following bits should be set in
6297          * order to enable memory self-refresh
6298          * The bit 22/21 of 0x42004
6299          * The bit 5 of 0x42020
6300          * The bit 15 of 0x45000
6301          */
6302         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6303                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
6304                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6305         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6306         I915_WRITE(DISP_ARB_CTL,
6307                    (I915_READ(DISP_ARB_CTL) |
6308                     DISP_FBC_WM_DIS));
6309
6310         /*
6311          * Based on the document from hardware guys the following bits
6312          * should be set unconditionally in order to enable FBC.
6313          * The bit 22 of 0x42000
6314          * The bit 22 of 0x42004
6315          * The bit 7,8,9 of 0x42020.
6316          */
6317         if (IS_IRONLAKE_M(dev_priv)) {
6318                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6319                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6320                            I915_READ(ILK_DISPLAY_CHICKEN1) |
6321                            ILK_FBCQ_DIS);
6322                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6323                            I915_READ(ILK_DISPLAY_CHICKEN2) |
6324                            ILK_DPARB_GATE);
6325         }
6326
6327         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6328
6329         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6330                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6331                    ILK_ELPIN_409_SELECT);
6332         I915_WRITE(_3D_CHICKEN2,
6333                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
6334                    _3D_CHICKEN2_WM_READ_PIPELINED);
6335
6336         /* WaDisableRenderCachePipelinedFlush:ilk */
6337         I915_WRITE(CACHE_MODE_0,
6338                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6339
6340         /* WaDisable_RenderCache_OperationalFlush:ilk */
6341         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6342
6343         g4x_disable_trickle_feed(dev_priv);
6344
6345         ibx_init_clock_gating(dev_priv);
6346 }
6347
6348 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6349 {
6350         enum pipe pipe;
6351         u32 val;
6352
6353         /*
6354          * On Ibex Peak and Cougar Point, we need to disable clock
6355          * gating for the panel power sequencer or it will fail to
6356          * start up when no ports are active.
6357          */
6358         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6359                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6360                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6361         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6362                    DPLS_EDP_PPS_FIX_DIS);
6363         /* The below fixes the weird display corruption, a few pixels shifted
6364          * downward, on (only) LVDS of some HP laptops with IVY.
6365          */
6366         for_each_pipe(dev_priv, pipe) {
6367                 val = I915_READ(TRANS_CHICKEN2(pipe));
6368                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6369                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6370                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6371                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6372                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
6373                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6374                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6375                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6376         }
6377         /* WADP0ClockGatingDisable */
6378         for_each_pipe(dev_priv, pipe) {
6379                 I915_WRITE(TRANS_CHICKEN1(pipe),
6380                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6381         }
6382 }
6383
6384 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6385 {
6386         u32 tmp;
6387
6388         tmp = I915_READ(MCH_SSKPD);
6389         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6390                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6391                               tmp);
6392 }
6393
6394 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6395 {
6396         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6397
6398         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6399
6400         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6401                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6402                    ILK_ELPIN_409_SELECT);
6403
6404         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
6405         I915_WRITE(_3D_CHICKEN,
6406                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
6407
6408         /* WaDisable_RenderCache_OperationalFlush:snb */
6409         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6410
6411         /*
6412          * BSpec recoomends 8x4 when MSAA is used,
6413          * however in practice 16x4 seems fastest.
6414          *
6415          * Note that PS/WM thread counts depend on the WIZ hashing
6416          * disable bit, which we don't touch here, but it's good
6417          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6418          */
6419         I915_WRITE(GEN6_GT_MODE,
6420                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6421
6422         I915_WRITE(CACHE_MODE_0,
6423                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6424
6425         I915_WRITE(GEN6_UCGCTL1,
6426                    I915_READ(GEN6_UCGCTL1) |
6427                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6428                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6429
6430         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
6431          * gating disable must be set.  Failure to set it results in
6432          * flickering pixels due to Z write ordering failures after
6433          * some amount of runtime in the Mesa "fire" demo, and Unigine
6434          * Sanctuary and Tropics, and apparently anything else with
6435          * alpha test or pixel discard.
6436          *
6437          * According to the spec, bit 11 (RCCUNIT) must also be set,
6438          * but we didn't debug actual testcases to find it out.
6439          *
6440          * WaDisableRCCUnitClockGating:snb
6441          * WaDisableRCPBUnitClockGating:snb
6442          */
6443         I915_WRITE(GEN6_UCGCTL2,
6444                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
6445                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
6446
6447         /* WaStripsFansDisableFastClipPerformanceFix:snb */
6448         I915_WRITE(_3D_CHICKEN3,
6449                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6450
6451         /*
6452          * Bspec says:
6453          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
6454          * 3DSTATE_SF number of SF output attributes is more than 16."
6455          */
6456         I915_WRITE(_3D_CHICKEN3,
6457                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
6458
6459         /*
6460          * According to the spec the following bits should be
6461          * set in order to enable memory self-refresh and fbc:
6462          * The bit21 and bit22 of 0x42000
6463          * The bit21 and bit22 of 0x42004
6464          * The bit5 and bit7 of 0x42020
6465          * The bit14 of 0x70180
6466          * The bit14 of 0x71180
6467          *
6468          * WaFbcAsynchFlipDisableFbcQueue:snb
6469          */
6470         I915_WRITE(ILK_DISPLAY_CHICKEN1,
6471                    I915_READ(ILK_DISPLAY_CHICKEN1) |
6472                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
6473         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6474                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6475                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
6476         I915_WRITE(ILK_DSPCLK_GATE_D,
6477                    I915_READ(ILK_DSPCLK_GATE_D) |
6478                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
6479                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6480
6481         g4x_disable_trickle_feed(dev_priv);
6482
6483         cpt_init_clock_gating(dev_priv);
6484
6485         gen6_check_mch_setup(dev_priv);
6486 }
6487
6488 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6489 {
6490         u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
6491
6492         /*
6493          * WaVSThreadDispatchOverride:ivb,vlv
6494          *
6495          * This actually overrides the dispatch
6496          * mode for all thread types.
6497          */
6498         reg &= ~GEN7_FF_SCHED_MASK;
6499         reg |= GEN7_FF_TS_SCHED_HW;
6500         reg |= GEN7_FF_VS_SCHED_HW;
6501         reg |= GEN7_FF_DS_SCHED_HW;
6502
6503         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
6504 }
6505
6506 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
6507 {
6508         /*
6509          * TODO: this bit should only be enabled when really needed, then
6510          * disabled when not needed anymore in order to save power.
6511          */
6512         if (HAS_PCH_LPT_LP(dev_priv))
6513                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
6514                            I915_READ(SOUTH_DSPCLK_GATE_D) |
6515                            PCH_LP_PARTITION_LEVEL_DISABLE);
6516
6517         /* WADPOClockGatingDisable:hsw */
6518         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
6519                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
6520                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6521 }
6522
6523 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
6524 {
6525         if (HAS_PCH_LPT_LP(dev_priv)) {
6526                 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
6527
6528                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
6529                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
6530         }
6531 }
6532
6533 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
6534                                    int general_prio_credits,
6535                                    int high_prio_credits)
6536 {
6537         u32 misccpctl;
6538         u32 val;
6539
6540         /* WaTempDisableDOPClkGating:bdw */
6541         misccpctl = I915_READ(GEN7_MISCCPCTL);
6542         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
6543
6544         val = I915_READ(GEN8_L3SQCREG1);
6545         val &= ~L3_PRIO_CREDITS_MASK;
6546         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
6547         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
6548         I915_WRITE(GEN8_L3SQCREG1, val);
6549
6550         /*
6551          * Wait at least 100 clocks before re-enabling clock gating.
6552          * See the definition of L3SQCREG1 in BSpec.
6553          */
6554         POSTING_READ(GEN8_L3SQCREG1);
6555         udelay(1);
6556         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
6557 }
6558
6559 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
6560 {
6561         /* This is not an Wa. Enable to reduce Sampler power */
6562         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
6563                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
6564
6565         /* WaEnable32PlaneMode:icl */
6566         I915_WRITE(GEN9_CSFE_CHICKEN1_RCS,
6567                    _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE));
6568
6569         /*
6570          * Wa_1408615072:icl,ehl  (vsunit)
6571          * Wa_1407596294:icl,ehl  (hsunit)
6572          */
6573         intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE,
6574                          0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
6575
6576         /* Wa_1407352427:icl,ehl */
6577         intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2,
6578                          0, PSDUNIT_CLKGATE_DIS);
6579 }
6580
6581 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
6582 {
6583         u32 vd_pg_enable = 0;
6584         unsigned int i;
6585
6586         /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
6587         for (i = 0; i < I915_MAX_VCS; i++) {
6588                 if (HAS_ENGINE(dev_priv, _VCS(i)))
6589                         vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
6590                                         VDN_MFX_POWERGATE_ENABLE(i);
6591         }
6592
6593         I915_WRITE(POWERGATE_ENABLE,
6594                    I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
6595 }
6596
6597 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
6598 {
6599         if (!HAS_PCH_CNP(dev_priv))
6600                 return;
6601
6602         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
6603         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
6604                    CNP_PWM_CGE_GATING_DISABLE);
6605 }
6606
6607 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
6608 {
6609         u32 val;
6610         cnp_init_clock_gating(dev_priv);
6611
6612         /* This is not an Wa. Enable for better image quality */
6613         I915_WRITE(_3D_CHICKEN3,
6614                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
6615
6616         /* WaEnableChickenDCPR:cnl */
6617         I915_WRITE(GEN8_CHICKEN_DCPR_1,
6618                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
6619
6620         /* WaFbcWakeMemOn:cnl */
6621         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
6622                    DISP_FBC_MEMORY_WAKE);
6623
6624         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
6625         /* ReadHitWriteOnlyDisable:cnl */
6626         val |= RCCUNIT_CLKGATE_DIS;
6627         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
6628         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
6629                 val |= SARBUNIT_CLKGATE_DIS;
6630         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
6631
6632         /* Wa_2201832410:cnl */
6633         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
6634         val |= GWUNIT_CLKGATE_DIS;
6635         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
6636
6637         /* WaDisableVFclkgate:cnl */
6638         /* WaVFUnitClockGatingDisable:cnl */
6639         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
6640         val |= VFUNIT_CLKGATE_DIS;
6641         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
6642 }
6643
6644 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
6645 {
6646         cnp_init_clock_gating(dev_priv);
6647         gen9_init_clock_gating(dev_priv);
6648
6649         /* WaFbcNukeOnHostModify:cfl */
6650         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6651                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
6652 }
6653
6654 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
6655 {
6656         gen9_init_clock_gating(dev_priv);
6657
6658         /* WaDisableSDEUnitClockGating:kbl */
6659         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
6660                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6661                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6662
6663         /* WaDisableGamClockGating:kbl */
6664         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
6665                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6666                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
6667
6668         /* WaFbcNukeOnHostModify:kbl */
6669         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6670                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
6671 }
6672
6673 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
6674 {
6675         gen9_init_clock_gating(dev_priv);
6676
6677         /* WAC6entrylatency:skl */
6678         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
6679                    FBC_LLC_FULLY_OPEN);
6680
6681         /* WaFbcNukeOnHostModify:skl */
6682         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6683                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
6684 }
6685
6686 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
6687 {
6688         enum pipe pipe;
6689
6690         /* WaSwitchSolVfFArbitrationPriority:bdw */
6691         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6692
6693         /* WaPsrDPAMaskVBlankInSRD:bdw */
6694         I915_WRITE(CHICKEN_PAR1_1,
6695                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
6696
6697         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
6698         for_each_pipe(dev_priv, pipe) {
6699                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
6700                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
6701                            BDW_DPRS_MASK_VBLANK_SRD);
6702         }
6703
6704         /* WaVSRefCountFullforceMissDisable:bdw */
6705         /* WaDSRefCountFullforceMissDisable:bdw */
6706         I915_WRITE(GEN7_FF_THREAD_MODE,
6707                    I915_READ(GEN7_FF_THREAD_MODE) &
6708                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6709
6710         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6711                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6712
6713         /* WaDisableSDEUnitClockGating:bdw */
6714         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6715                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6716
6717         /* WaProgramL3SqcReg1Default:bdw */
6718         gen8_set_l3sqc_credits(dev_priv, 30, 2);
6719
6720         /* WaKVMNotificationOnConfigChange:bdw */
6721         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
6722                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
6723
6724         lpt_init_clock_gating(dev_priv);
6725
6726         /* WaDisableDopClockGating:bdw
6727          *
6728          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
6729          * clock gating.
6730          */
6731         I915_WRITE(GEN6_UCGCTL1,
6732                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
6733 }
6734
6735 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
6736 {
6737         /* L3 caching of data atomics doesn't work -- disable it. */
6738         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
6739         I915_WRITE(HSW_ROW_CHICKEN3,
6740                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
6741
6742         /* This is required by WaCatErrorRejectionIssue:hsw */
6743         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6744                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6745                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6746
6747         /* WaVSRefCountFullforceMissDisable:hsw */
6748         I915_WRITE(GEN7_FF_THREAD_MODE,
6749                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
6750
6751         /* WaDisable_RenderCache_OperationalFlush:hsw */
6752         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6753
6754         /* enable HiZ Raw Stall Optimization */
6755         I915_WRITE(CACHE_MODE_0_GEN7,
6756                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6757
6758         /* WaDisable4x2SubspanOptimization:hsw */
6759         I915_WRITE(CACHE_MODE_1,
6760                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6761
6762         /*
6763          * BSpec recommends 8x4 when MSAA is used,
6764          * however in practice 16x4 seems fastest.
6765          *
6766          * Note that PS/WM thread counts depend on the WIZ hashing
6767          * disable bit, which we don't touch here, but it's good
6768          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6769          */
6770         I915_WRITE(GEN7_GT_MODE,
6771                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6772
6773         /* WaSampleCChickenBitEnable:hsw */
6774         I915_WRITE(HALF_SLICE_CHICKEN3,
6775                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
6776
6777         /* WaSwitchSolVfFArbitrationPriority:hsw */
6778         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6779
6780         lpt_init_clock_gating(dev_priv);
6781 }
6782
6783 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
6784 {
6785         u32 snpcr;
6786
6787         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6788
6789         /* WaDisableEarlyCull:ivb */
6790         I915_WRITE(_3D_CHICKEN3,
6791                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6792
6793         /* WaDisableBackToBackFlipFix:ivb */
6794         I915_WRITE(IVB_CHICKEN3,
6795                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6796                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6797
6798         /* WaDisablePSDDualDispatchEnable:ivb */
6799         if (IS_IVB_GT1(dev_priv))
6800                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6801                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6802
6803         /* WaDisable_RenderCache_OperationalFlush:ivb */
6804         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6805
6806         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6807         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
6808                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
6809
6810         /* WaApplyL3ControlAndL3ChickenMode:ivb */
6811         I915_WRITE(GEN7_L3CNTLREG1,
6812                         GEN7_WA_FOR_GEN7_L3_CONTROL);
6813         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6814                    GEN7_WA_L3_CHICKEN_MODE);
6815         if (IS_IVB_GT1(dev_priv))
6816                 I915_WRITE(GEN7_ROW_CHICKEN2,
6817                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6818         else {
6819                 /* must write both registers */
6820                 I915_WRITE(GEN7_ROW_CHICKEN2,
6821                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6822                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
6823                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6824         }
6825
6826         /* WaForceL3Serialization:ivb */
6827         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6828                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6829
6830         /*
6831          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6832          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
6833          */
6834         I915_WRITE(GEN6_UCGCTL2,
6835                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6836
6837         /* This is required by WaCatErrorRejectionIssue:ivb */
6838         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6839                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6840                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6841
6842         g4x_disable_trickle_feed(dev_priv);
6843
6844         gen7_setup_fixed_func_scheduler(dev_priv);
6845
6846         if (0) { /* causes HiZ corruption on ivb:gt1 */
6847                 /* enable HiZ Raw Stall Optimization */
6848                 I915_WRITE(CACHE_MODE_0_GEN7,
6849                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6850         }
6851
6852         /* WaDisable4x2SubspanOptimization:ivb */
6853         I915_WRITE(CACHE_MODE_1,
6854                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6855
6856         /*
6857          * BSpec recommends 8x4 when MSAA is used,
6858          * however in practice 16x4 seems fastest.
6859          *
6860          * Note that PS/WM thread counts depend on the WIZ hashing
6861          * disable bit, which we don't touch here, but it's good
6862          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6863          */
6864         I915_WRITE(GEN7_GT_MODE,
6865                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6866
6867         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
6868         snpcr &= ~GEN6_MBC_SNPCR_MASK;
6869         snpcr |= GEN6_MBC_SNPCR_MED;
6870         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
6871
6872         if (!HAS_PCH_NOP(dev_priv))
6873                 cpt_init_clock_gating(dev_priv);
6874
6875         gen6_check_mch_setup(dev_priv);
6876 }
6877
6878 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
6879 {
6880         /* WaDisableEarlyCull:vlv */
6881         I915_WRITE(_3D_CHICKEN3,
6882                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6883
6884         /* WaDisableBackToBackFlipFix:vlv */
6885         I915_WRITE(IVB_CHICKEN3,
6886                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6887                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6888
6889         /* WaPsdDispatchEnable:vlv */
6890         /* WaDisablePSDDualDispatchEnable:vlv */
6891         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6892                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
6893                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6894
6895         /* WaDisable_RenderCache_OperationalFlush:vlv */
6896         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6897
6898         /* WaForceL3Serialization:vlv */
6899         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6900                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6901
6902         /* WaDisableDopClockGating:vlv */
6903         I915_WRITE(GEN7_ROW_CHICKEN2,
6904                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6905
6906         /* This is required by WaCatErrorRejectionIssue:vlv */
6907         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6908                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6909                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6910
6911         gen7_setup_fixed_func_scheduler(dev_priv);
6912
6913         /*
6914          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6915          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
6916          */
6917         I915_WRITE(GEN6_UCGCTL2,
6918                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6919
6920         /* WaDisableL3Bank2xClockGate:vlv
6921          * Disabling L3 clock gating- MMIO 940c[25] = 1
6922          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
6923         I915_WRITE(GEN7_UCGCTL4,
6924                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
6925
6926         /*
6927          * BSpec says this must be set, even though
6928          * WaDisable4x2SubspanOptimization isn't listed for VLV.
6929          */
6930         I915_WRITE(CACHE_MODE_1,
6931                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6932
6933         /*
6934          * BSpec recommends 8x4 when MSAA is used,
6935          * however in practice 16x4 seems fastest.
6936          *
6937          * Note that PS/WM thread counts depend on the WIZ hashing
6938          * disable bit, which we don't touch here, but it's good
6939          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6940          */
6941         I915_WRITE(GEN7_GT_MODE,
6942                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6943
6944         /*
6945          * WaIncreaseL3CreditsForVLVB0:vlv
6946          * This is the hardware default actually.
6947          */
6948         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
6949
6950         /*
6951          * WaDisableVLVClockGating_VBIIssue:vlv
6952          * Disable clock gating on th GCFG unit to prevent a delay
6953          * in the reporting of vblank events.
6954          */
6955         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6956 }
6957
6958 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
6959 {
6960         /* WaVSRefCountFullforceMissDisable:chv */
6961         /* WaDSRefCountFullforceMissDisable:chv */
6962         I915_WRITE(GEN7_FF_THREAD_MODE,
6963                    I915_READ(GEN7_FF_THREAD_MODE) &
6964                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6965
6966         /* WaDisableSemaphoreAndSyncFlipWait:chv */
6967         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6968                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6969
6970         /* WaDisableCSUnitClockGating:chv */
6971         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6972                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6973
6974         /* WaDisableSDEUnitClockGating:chv */
6975         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6976                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6977
6978         /*
6979          * WaProgramL3SqcReg1Default:chv
6980          * See gfxspecs/Related Documents/Performance Guide/
6981          * LSQC Setting Recommendations.
6982          */
6983         gen8_set_l3sqc_credits(dev_priv, 38, 2);
6984 }
6985
6986 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
6987 {
6988         u32 dspclk_gate;
6989
6990         I915_WRITE(RENCLK_GATE_D1, 0);
6991         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
6992                    GS_UNIT_CLOCK_GATE_DISABLE |
6993                    CL_UNIT_CLOCK_GATE_DISABLE);
6994         I915_WRITE(RAMCLK_GATE_D, 0);
6995         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
6996                 OVRUNIT_CLOCK_GATE_DISABLE |
6997                 OVCUNIT_CLOCK_GATE_DISABLE;
6998         if (IS_GM45(dev_priv))
6999                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7000         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7001
7002         /* WaDisableRenderCachePipelinedFlush */
7003         I915_WRITE(CACHE_MODE_0,
7004                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
7005
7006         /* WaDisable_RenderCache_OperationalFlush:g4x */
7007         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7008
7009         g4x_disable_trickle_feed(dev_priv);
7010 }
7011
7012 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7013 {
7014         struct intel_uncore *uncore = &dev_priv->uncore;
7015
7016         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7017         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7018         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7019         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7020         intel_uncore_write16(uncore, DEUC, 0);
7021         intel_uncore_write(uncore,
7022                            MI_ARB_STATE,
7023                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7024
7025         /* WaDisable_RenderCache_OperationalFlush:gen4 */
7026         intel_uncore_write(uncore,
7027                            CACHE_MODE_0,
7028                            _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7029 }
7030
7031 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7032 {
7033         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7034                    I965_RCC_CLOCK_GATE_DISABLE |
7035                    I965_RCPB_CLOCK_GATE_DISABLE |
7036                    I965_ISC_CLOCK_GATE_DISABLE |
7037                    I965_FBC_CLOCK_GATE_DISABLE);
7038         I915_WRITE(RENCLK_GATE_D2, 0);
7039         I915_WRITE(MI_ARB_STATE,
7040                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7041
7042         /* WaDisable_RenderCache_OperationalFlush:gen4 */
7043         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7044 }
7045
7046 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7047 {
7048         u32 dstate = I915_READ(D_STATE);
7049
7050         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7051                 DSTATE_DOT_CLOCK_GATING;
7052         I915_WRITE(D_STATE, dstate);
7053
7054         if (IS_PINEVIEW(dev_priv))
7055                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7056
7057         /* IIR "flip pending" means done if this bit is set */
7058         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7059
7060         /* interrupts should cause a wake up from C3 */
7061         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7062
7063         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7064         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7065
7066         I915_WRITE(MI_ARB_STATE,
7067                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7068 }
7069
7070 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7071 {
7072         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7073
7074         /* interrupts should cause a wake up from C3 */
7075         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7076                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7077
7078         I915_WRITE(MEM_MODE,
7079                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7080 }
7081
7082 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7083 {
7084         I915_WRITE(MEM_MODE,
7085                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7086                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7087 }
7088
7089 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7090 {
7091         dev_priv->display.init_clock_gating(dev_priv);
7092 }
7093
7094 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7095 {
7096         if (HAS_PCH_LPT(dev_priv))
7097                 lpt_suspend_hw(dev_priv);
7098 }
7099
7100 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7101 {
7102         DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
7103 }
7104
7105 /**
7106  * intel_init_clock_gating_hooks - setup the clock gating hooks
7107  * @dev_priv: device private
7108  *
7109  * Setup the hooks that configure which clocks of a given platform can be
7110  * gated and also apply various GT and display specific workarounds for these
7111  * platforms. Note that some GT specific workarounds are applied separately
7112  * when GPU contexts or batchbuffers start their execution.
7113  */
7114 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7115 {
7116         if (IS_GEN(dev_priv, 12))
7117                 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7118         else if (IS_GEN(dev_priv, 11))
7119                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7120         else if (IS_CANNONLAKE(dev_priv))
7121                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7122         else if (IS_COFFEELAKE(dev_priv))
7123                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7124         else if (IS_SKYLAKE(dev_priv))
7125                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7126         else if (IS_KABYLAKE(dev_priv))
7127                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7128         else if (IS_BROXTON(dev_priv))
7129                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7130         else if (IS_GEMINILAKE(dev_priv))
7131                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7132         else if (IS_BROADWELL(dev_priv))
7133                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7134         else if (IS_CHERRYVIEW(dev_priv))
7135                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7136         else if (IS_HASWELL(dev_priv))
7137                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7138         else if (IS_IVYBRIDGE(dev_priv))
7139                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7140         else if (IS_VALLEYVIEW(dev_priv))
7141                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7142         else if (IS_GEN(dev_priv, 6))
7143                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7144         else if (IS_GEN(dev_priv, 5))
7145                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7146         else if (IS_G4X(dev_priv))
7147                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7148         else if (IS_I965GM(dev_priv))
7149                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7150         else if (IS_I965G(dev_priv))
7151                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7152         else if (IS_GEN(dev_priv, 3))
7153                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7154         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7155                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7156         else if (IS_GEN(dev_priv, 2))
7157                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7158         else {
7159                 MISSING_CASE(INTEL_DEVID(dev_priv));
7160                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7161         }
7162 }
7163
7164 /* Set up chip specific power management-related functions */
7165 void intel_init_pm(struct drm_i915_private *dev_priv)
7166 {
7167         /* For cxsr */
7168         if (IS_PINEVIEW(dev_priv))
7169                 i915_pineview_get_mem_freq(dev_priv);
7170         else if (IS_GEN(dev_priv, 5))
7171                 i915_ironlake_get_mem_freq(dev_priv);
7172
7173         if (intel_has_sagv(dev_priv))
7174                 skl_setup_sagv_block_time(dev_priv);
7175
7176         /* For FIFO watermark updates */
7177         if (INTEL_GEN(dev_priv) >= 9) {
7178                 skl_setup_wm_latency(dev_priv);
7179                 dev_priv->display.initial_watermarks = skl_initial_wm;
7180                 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
7181                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7182         } else if (HAS_PCH_SPLIT(dev_priv)) {
7183                 ilk_setup_wm_latency(dev_priv);
7184
7185                 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7186                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7187                     (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7188                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7189                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7190                         dev_priv->display.compute_intermediate_wm =
7191                                 ilk_compute_intermediate_wm;
7192                         dev_priv->display.initial_watermarks =
7193                                 ilk_initial_watermarks;
7194                         dev_priv->display.optimize_watermarks =
7195                                 ilk_optimize_watermarks;
7196                 } else {
7197                         DRM_DEBUG_KMS("Failed to read display plane latency. "
7198                                       "Disable CxSR\n");
7199                 }
7200         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7201                 vlv_setup_wm_latency(dev_priv);
7202                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7203                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7204                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7205                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7206                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7207         } else if (IS_G4X(dev_priv)) {
7208                 g4x_setup_wm_latency(dev_priv);
7209                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7210                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7211                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7212                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7213         } else if (IS_PINEVIEW(dev_priv)) {
7214                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7215                                             dev_priv->is_ddr3,
7216                                             dev_priv->fsb_freq,
7217                                             dev_priv->mem_freq)) {
7218                         DRM_INFO("failed to find known CxSR latency "
7219                                  "(found ddr%s fsb freq %d, mem freq %d), "
7220                                  "disabling CxSR\n",
7221                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7222                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7223                         /* Disable CxSR and never update its watermark again */
7224                         intel_set_memory_cxsr(dev_priv, false);
7225                         dev_priv->display.update_wm = NULL;
7226                 } else
7227                         dev_priv->display.update_wm = pineview_update_wm;
7228         } else if (IS_GEN(dev_priv, 4)) {
7229                 dev_priv->display.update_wm = i965_update_wm;
7230         } else if (IS_GEN(dev_priv, 3)) {
7231                 dev_priv->display.update_wm = i9xx_update_wm;
7232                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7233         } else if (IS_GEN(dev_priv, 2)) {
7234                 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7235                         dev_priv->display.update_wm = i845_update_wm;
7236                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7237                 } else {
7238                         dev_priv->display.update_wm = i9xx_update_wm;
7239                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7240                 }
7241         } else {
7242                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
7243         }
7244 }
7245
7246 void intel_pm_setup(struct drm_i915_private *dev_priv)
7247 {
7248         dev_priv->runtime_pm.suspended = false;
7249         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7250 }