drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include <drm/drm_atomic_helper.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include "display/intel_atomic.h"
  36 #include "display/intel_bw.h"
  37 #include "display/intel_display_types.h"
  38 #include "display/intel_fbc.h"
  39 #include "display/intel_sprite.h"
  40
  41 #include "gt/intel_llc.h"
  42
  43 #include "i915_drv.h"
  44 #include "i915_fixed.h"
  45 #include "i915_irq.h"
  46 #include "i915_trace.h"
  47 #include "intel_pm.h"
  48 #include "intel_sideband.h"
  49 #include "../../../platform/x86/intel_ips.h"
  50
  51 /* Stores plane specific WM parameters */
  52 struct skl_wm_params {
  53         bool x_tiled, y_tiled;
  54         bool rc_surface;
  55         bool is_planar;
  56         u32 width;
  57         u8 cpp;
  58         u32 plane_pixel_rate;
  59         u32 y_min_scanlines;
  60         u32 plane_bytes_per_line;
  61         uint_fixed_16_16_t plane_blocks_per_line;
  62         uint_fixed_16_16_t y_tile_minimum;
  63         u32 linetime_us;
  64         u32 dbuf_block_size;
  65 };
  66
  67 /* used in computing the new watermarks state */
  68 struct intel_wm_config {
  69         unsigned int num_pipes_active;
  70         bool sprites_enabled;
  71         bool sprites_scaled;
  72 };
  73
  74 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  75 {
  76         if (HAS_LLC(dev_priv)) {
  77                 /*
  78                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  79                  * Display WA #0390: skl,kbl
  80                  *
  81                  * Must match Sampler, Pixel Back End, and Media. See
  82                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  83                  */
  84                 I915_WRITE(CHICKEN_PAR1_1,
  85                            I915_READ(CHICKEN_PAR1_1) |
  86                            SKL_DE_COMPRESSED_HASH_MODE);
  87         }
  88
  89         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  90         I915_WRITE(CHICKEN_PAR1_1,
  91                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  92
  93         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  94         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  95                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  96
  97         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  98         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  99                    DISP_FBC_MEMORY_WAKE);
 100
 101         if (IS_SKYLAKE(dev_priv)) {
 102                 /* WaDisableDopClockGating */
 103                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
 104                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 105         }
 106 }
 107
 108 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 109 {
 110         gen9_init_clock_gating(dev_priv);
 111
 112         /* WaDisableSDEUnitClockGating:bxt */
 113         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 114                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 115
 116         /*
 117          * FIXME:
 118          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 119          */
 120         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 121                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 122
 123         /*
 124          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 125          * to stay fully on.
 126          */
 127         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 128                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 129
 130         /*
 131          * Lower the display internal timeout.
 132          * This is needed to avoid any hard hangs when DSI port PLL
 133          * is off and a MMIO access is attempted by any privilege
 134          * application, using batch buffers or any other means.
 135          */
 136         I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 137
 138         /* WaFbcTurnOffFbcWatermark:bxt */
 139         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 140                    DISP_FBC_WM_DIS);
 141
 142         /* WaFbcHighMemBwCorruptionAvoidance:bxt */
 143         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
 144                    ILK_DPFC_DISABLE_DUMMY0);
 145 }
 146
 147 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 148 {
 149         gen9_init_clock_gating(dev_priv);
 150
 151         /*
 152          * WaDisablePWMClockGating:glk
 153          * Backlight PWM may stop in the asserted state, causing backlight
 154          * to stay fully on.
 155          */
 156         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 157                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 158 }
 159
 160 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 161 {
 162         u32 tmp;
 163
 164         tmp = I915_READ(CLKCFG);
 165
 166         switch (tmp & CLKCFG_FSB_MASK) {
 167         case CLKCFG_FSB_533:
 168                 dev_priv->fsb_freq = 533; /* 133*4 */
 169                 break;
 170         case CLKCFG_FSB_800:
 171                 dev_priv->fsb_freq = 800; /* 200*4 */
 172                 break;
 173         case CLKCFG_FSB_667:
 174                 dev_priv->fsb_freq =  667; /* 167*4 */
 175                 break;
 176         case CLKCFG_FSB_400:
 177                 dev_priv->fsb_freq = 400; /* 100*4 */
 178                 break;
 179         }
 180
 181         switch (tmp & CLKCFG_MEM_MASK) {
 182         case CLKCFG_MEM_533:
 183                 dev_priv->mem_freq = 533;
 184                 break;
 185         case CLKCFG_MEM_667:
 186                 dev_priv->mem_freq = 667;
 187                 break;
 188         case CLKCFG_MEM_800:
 189                 dev_priv->mem_freq = 800;
 190                 break;
 191         }
 192
 193         /* detect pineview DDR3 setting */
 194         tmp = I915_READ(CSHRDDR3CTL);
 195         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 196 }
 197
 198 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 199 {
 200         u16 ddrpll, csipll;
 201
 202         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 203         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 204
 205         switch (ddrpll & 0xff) {
 206         case 0xc:
 207                 dev_priv->mem_freq = 800;
 208                 break;
 209         case 0x10:
 210                 dev_priv->mem_freq = 1066;
 211                 break;
 212         case 0x14:
 213                 dev_priv->mem_freq = 1333;
 214                 break;
 215         case 0x18:
 216                 dev_priv->mem_freq = 1600;
 217                 break;
 218         default:
 219                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 220                         ddrpll & 0xff);
 221                 dev_priv->mem_freq = 0;
 222                 break;
 223         }
 224
 225         switch (csipll & 0x3ff) {
 226         case 0x00c:
 227                 dev_priv->fsb_freq = 3200;
 228                 break;
 229         case 0x00e:
 230                 dev_priv->fsb_freq = 3733;
 231                 break;
 232         case 0x010:
 233                 dev_priv->fsb_freq = 4266;
 234                 break;
 235         case 0x012:
 236                 dev_priv->fsb_freq = 4800;
 237                 break;
 238         case 0x014:
 239                 dev_priv->fsb_freq = 5333;
 240                 break;
 241         case 0x016:
 242                 dev_priv->fsb_freq = 5866;
 243                 break;
 244         case 0x018:
 245                 dev_priv->fsb_freq = 6400;
 246                 break;
 247         default:
 248                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 249                         csipll & 0x3ff);
 250                 dev_priv->fsb_freq = 0;
 251                 break;
 252         }
 253 }
 254
 255 static const struct cxsr_latency cxsr_latency_table[] = {
 256         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 257         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 258         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 259         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 260         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 261
 262         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 263         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 264         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 265         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 266         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 267
 268         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 269         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 270         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 271         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 272         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 273
 274         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 275         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 276         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 277         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 278         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 279
 280         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 281         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 282         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 283         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 284         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 285
 286         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 287         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 288         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 289         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 290         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 291 };
 292
 293 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 294                                                          bool is_ddr3,
 295                                                          int fsb,
 296                                                          int mem)
 297 {
 298         const struct cxsr_latency *latency;
 299         int i;
 300
 301         if (fsb == 0 || mem == 0)
 302                 return NULL;
 303
 304         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 305                 latency = &cxsr_latency_table[i];
 306                 if (is_desktop == latency->is_desktop &&
 307                     is_ddr3 == latency->is_ddr3 &&
 308                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 309                         return latency;
 310         }
 311
 312         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 313
 314         return NULL;
 315 }
 316
 317 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 318 {
 319         u32 val;
 320
 321         vlv_punit_get(dev_priv);
 322
 323         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 324         if (enable)
 325                 val &= ~FORCE_DDR_HIGH_FREQ;
 326         else
 327                 val |= FORCE_DDR_HIGH_FREQ;
 328         val &= ~FORCE_DDR_LOW_FREQ;
 329         val |= FORCE_DDR_FREQ_REQ_ACK;
 330         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 331
 332         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 333                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 334                 drm_err(&dev_priv->drm,
 335                         "timed out waiting for Punit DDR DVFS request\n");
 336
 337         vlv_punit_put(dev_priv);
 338 }
 339
 340 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 341 {
 342         u32 val;
 343
 344         vlv_punit_get(dev_priv);
 345
 346         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 347         if (enable)
 348                 val |= DSP_MAXFIFO_PM5_ENABLE;
 349         else
 350                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 351         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 352
 353         vlv_punit_put(dev_priv);
 354 }
 355
 356 #define FW_WM(value, plane) \
 357         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 358
 359 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 360 {
 361         bool was_enabled;
 362         u32 val;
 363
 364         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 365                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 366                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 367                 POSTING_READ(FW_BLC_SELF_VLV);
 368         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 369                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 370                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 371                 POSTING_READ(FW_BLC_SELF);
 372         } else if (IS_PINEVIEW(dev_priv)) {
 373                 val = I915_READ(DSPFW3);
 374                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 375                 if (enable)
 376                         val |= PINEVIEW_SELF_REFRESH_EN;
 377                 else
 378                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 379                 I915_WRITE(DSPFW3, val);
 380                 POSTING_READ(DSPFW3);
 381         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 382                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 383                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 384                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 385                 I915_WRITE(FW_BLC_SELF, val);
 386                 POSTING_READ(FW_BLC_SELF);
 387         } else if (IS_I915GM(dev_priv)) {
 388                 /*
 389                  * FIXME can't find a bit like this for 915G, and
 390                  * and yet it does have the related watermark in
 391                  * FW_BLC_SELF. What's going on?
 392                  */
 393                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 394                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 395                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 396                 I915_WRITE(INSTPM, val);
 397                 POSTING_READ(INSTPM);
 398         } else {
 399                 return false;
 400         }
 401
 402         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 403
 404         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 405                     enableddisabled(enable),
 406                     enableddisabled(was_enabled));
 407
 408         return was_enabled;
 409 }
 410
 411 /**
 412  * intel_set_memory_cxsr - Configure CxSR state
 413  * @dev_priv: i915 device
 414  * @enable: Allow vs. disallow CxSR
 415  *
 416  * Allow or disallow the system to enter a special CxSR
 417  * (C-state self refresh) state. What typically happens in CxSR mode
 418  * is that several display FIFOs may get combined into a single larger
 419  * FIFO for a particular plane (so called max FIFO mode) to allow the
 420  * system to defer memory fetches longer, and the memory will enter
 421  * self refresh.
 422  *
 423  * Note that enabling CxSR does not guarantee that the system enter
 424  * this special mode, nor does it guarantee that the system stays
 425  * in that mode once entered. So this just allows/disallows the system
 426  * to autonomously utilize the CxSR mode. Other factors such as core
 427  * C-states will affect when/if the system actually enters/exits the
 428  * CxSR mode.
 429  *
 430  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 431  * and the system is free to enter/exit memory self refresh at any time
 432  * even when the use of CxSR has been disallowed.
 433  *
 434  * While the system is actually in the CxSR/max FIFO mode, some plane
 435  * control registers will not get latched on vblank. Thus in order to
 436  * guarantee the system will respond to changes in the plane registers
 437  * we must always disallow CxSR prior to making changes to those registers.
 438  * Unfortunately the system will re-evaluate the CxSR conditions at
 439  * frame start which happens after vblank start (which is when the plane
 440  * registers would get latched), so we can't proceed with the plane update
 441  * during the same frame where we disallowed CxSR.
 442  *
 443  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 444  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 445  * the hardware w.r.t. HPLL SR when writing to plane registers.
 446  * Disallowing just CxSR is sufficient.
 447  */
 448 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 449 {
 450         bool ret;
 451
 452         mutex_lock(&dev_priv->wm.wm_mutex);
 453         ret = _intel_set_memory_cxsr(dev_priv, enable);
 454         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 455                 dev_priv->wm.vlv.cxsr = enable;
 456         else if (IS_G4X(dev_priv))
 457                 dev_priv->wm.g4x.cxsr = enable;
 458         mutex_unlock(&dev_priv->wm.wm_mutex);
 459
 460         return ret;
 461 }
 462
 463 /*
 464  * Latency for FIFO fetches is dependent on several factors:
 465  *   - memory configuration (speed, channels)
 466  *   - chipset
 467  *   - current MCH state
 468  * It can be fairly high in some situations, so here we assume a fairly
 469  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 470  * set this value too high, the FIFO will fetch frequently to stay full)
 471  * and power consumption (set it too low to save power and we might see
 472  * FIFO underruns and display "flicker").
 473  *
 474  * A value of 5us seems to be a good balance; safe for very low end
 475  * platforms but not overly aggressive on lower latency configs.
 476  */
 477 static const int pessimal_latency_ns = 5000;
 478
 479 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 480         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 481
 482 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 483 {
 484         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 485         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 486         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 487         enum pipe pipe = crtc->pipe;
 488         int sprite0_start, sprite1_start;
 489         u32 dsparb, dsparb2, dsparb3;
 490
 491         switch (pipe) {
 492         case PIPE_A:
 493                 dsparb = I915_READ(DSPARB);
 494                 dsparb2 = I915_READ(DSPARB2);
 495                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 496                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 497                 break;
 498         case PIPE_B:
 499                 dsparb = I915_READ(DSPARB);
 500                 dsparb2 = I915_READ(DSPARB2);
 501                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 502                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 503                 break;
 504         case PIPE_C:
 505                 dsparb2 = I915_READ(DSPARB2);
 506                 dsparb3 = I915_READ(DSPARB3);
 507                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 508                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 509                 break;
 510         default:
 511                 MISSING_CASE(pipe);
 512                 return;
 513         }
 514
 515         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 516         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 517         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 518         fifo_state->plane[PLANE_CURSOR] = 63;
 519 }
 520
 521 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 522                               enum i9xx_plane_id i9xx_plane)
 523 {
 524         u32 dsparb = I915_READ(DSPARB);
 525         int size;
 526
 527         size = dsparb & 0x7f;
 528         if (i9xx_plane == PLANE_B)
 529                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 530
 531         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 532                     dsparb, plane_name(i9xx_plane), size);
 533
 534         return size;
 535 }
 536
 537 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 538                               enum i9xx_plane_id i9xx_plane)
 539 {
 540         u32 dsparb = I915_READ(DSPARB);
 541         int size;
 542
 543         size = dsparb & 0x1ff;
 544         if (i9xx_plane == PLANE_B)
 545                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 546         size >>= 1; /* Convert to cachelines */
 547
 548         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 549                     dsparb, plane_name(i9xx_plane), size);
 550
 551         return size;
 552 }
 553
 554 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 555                               enum i9xx_plane_id i9xx_plane)
 556 {
 557         u32 dsparb = I915_READ(DSPARB);
 558         int size;
 559
 560         size = dsparb & 0x7f;
 561         size >>= 2; /* Convert to cachelines */
 562
 563         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 564                     dsparb, plane_name(i9xx_plane), size);
 565
 566         return size;
 567 }
 568
 569 /* Pineview has different values for various configs */
 570 static const struct intel_watermark_params pnv_display_wm = {
 571         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 572         .max_wm = PINEVIEW_MAX_WM,
 573         .default_wm = PINEVIEW_DFT_WM,
 574         .guard_size = PINEVIEW_GUARD_WM,
 575         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 576 };
 577
 578 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 579         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 580         .max_wm = PINEVIEW_MAX_WM,
 581         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 582         .guard_size = PINEVIEW_GUARD_WM,
 583         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 584 };
 585
 586 static const struct intel_watermark_params pnv_cursor_wm = {
 587         .fifo_size = PINEVIEW_CURSOR_FIFO,
 588         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 589         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 590         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 591         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 592 };
 593
 594 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 595         .fifo_size = PINEVIEW_CURSOR_FIFO,
 596         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 597         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 598         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 599         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 600 };
 601
 602 static const struct intel_watermark_params i965_cursor_wm_info = {
 603         .fifo_size = I965_CURSOR_FIFO,
 604         .max_wm = I965_CURSOR_MAX_WM,
 605         .default_wm = I965_CURSOR_DFT_WM,
 606         .guard_size = 2,
 607         .cacheline_size = I915_FIFO_LINE_SIZE,
 608 };
 609
 610 static const struct intel_watermark_params i945_wm_info = {
 611         .fifo_size = I945_FIFO_SIZE,
 612         .max_wm = I915_MAX_WM,
 613         .default_wm = 1,
 614         .guard_size = 2,
 615         .cacheline_size = I915_FIFO_LINE_SIZE,
 616 };
 617
 618 static const struct intel_watermark_params i915_wm_info = {
 619         .fifo_size = I915_FIFO_SIZE,
 620         .max_wm = I915_MAX_WM,
 621         .default_wm = 1,
 622         .guard_size = 2,
 623         .cacheline_size = I915_FIFO_LINE_SIZE,
 624 };
 625
 626 static const struct intel_watermark_params i830_a_wm_info = {
 627         .fifo_size = I855GM_FIFO_SIZE,
 628         .max_wm = I915_MAX_WM,
 629         .default_wm = 1,
 630         .guard_size = 2,
 631         .cacheline_size = I830_FIFO_LINE_SIZE,
 632 };
 633
 634 static const struct intel_watermark_params i830_bc_wm_info = {
 635         .fifo_size = I855GM_FIFO_SIZE,
 636         .max_wm = I915_MAX_WM/2,
 637         .default_wm = 1,
 638         .guard_size = 2,
 639         .cacheline_size = I830_FIFO_LINE_SIZE,
 640 };
 641
 642 static const struct intel_watermark_params i845_wm_info = {
 643         .fifo_size = I830_FIFO_SIZE,
 644         .max_wm = I915_MAX_WM,
 645         .default_wm = 1,
 646         .guard_size = 2,
 647         .cacheline_size = I830_FIFO_LINE_SIZE,
 648 };
 649
 650 /**
 651  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 652  * @pixel_rate: Pipe pixel rate in kHz
 653  * @cpp: Plane bytes per pixel
 654  * @latency: Memory wakeup latency in 0.1us units
 655  *
 656  * Compute the watermark using the method 1 or "small buffer"
 657  * formula. The caller may additonally add extra cachelines
 658  * to account for TLB misses and clock crossings.
 659  *
 660  * This method is concerned with the short term drain rate
 661  * of the FIFO, ie. it does not account for blanking periods
 662  * which would effectively reduce the average drain rate across
 663  * a longer period. The name "small" refers to the fact the
 664  * FIFO is relatively small compared to the amount of data
 665  * fetched.
 666  *
 667  * The FIFO level vs. time graph might look something like:
 668  *
 669  *   |\   |\
 670  *   | \  | \
 671  * __---__---__ (- plane active, _ blanking)
 672  * -> time
 673  *
 674  * or perhaps like this:
 675  *
 676  *   |\|\  |\|\
 677  * __----__----__ (- plane active, _ blanking)
 678  * -> time
 679  *
 680  * Returns:
 681  * The watermark in bytes
 682  */
 683 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 684                                      unsigned int cpp,
 685                                      unsigned int latency)
 686 {
 687         u64 ret;
 688
 689         ret = mul_u32_u32(pixel_rate, cpp * latency);
 690         ret = DIV_ROUND_UP_ULL(ret, 10000);
 691
 692         return ret;
 693 }
 694
 695 /**
 696  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 697  * @pixel_rate: Pipe pixel rate in kHz
 698  * @htotal: Pipe horizontal total
 699  * @width: Plane width in pixels
 700  * @cpp: Plane bytes per pixel
 701  * @latency: Memory wakeup latency in 0.1us units
 702  *
 703  * Compute the watermark using the method 2 or "large buffer"
 704  * formula. The caller may additonally add extra cachelines
 705  * to account for TLB misses and clock crossings.
 706  *
 707  * This method is concerned with the long term drain rate
 708  * of the FIFO, ie. it does account for blanking periods
 709  * which effectively reduce the average drain rate across
 710  * a longer period. The name "large" refers to the fact the
 711  * FIFO is relatively large compared to the amount of data
 712  * fetched.
 713  *
 714  * The FIFO level vs. time graph might look something like:
 715  *
 716  *    |\___       |\___
 717  *    |    \___   |    \___
 718  *    |        \  |        \
 719  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 720  * -> time
 721  *
 722  * Returns:
 723  * The watermark in bytes
 724  */
 725 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 726                                      unsigned int htotal,
 727                                      unsigned int width,
 728                                      unsigned int cpp,
 729                                      unsigned int latency)
 730 {
 731         unsigned int ret;
 732
 733         /*
 734          * FIXME remove once all users are computing
 735          * watermarks in the correct place.
 736          */
 737         if (WARN_ON_ONCE(htotal == 0))
 738                 htotal = 1;
 739
 740         ret = (latency * pixel_rate) / (htotal * 10000);
 741         ret = (ret + 1) * width * cpp;
 742
 743         return ret;
 744 }
 745
 746 /**
 747  * intel_calculate_wm - calculate watermark level
 748  * @pixel_rate: pixel clock
 749  * @wm: chip FIFO params
 750  * @fifo_size: size of the FIFO buffer
 751  * @cpp: bytes per pixel
 752  * @latency_ns: memory latency for the platform
 753  *
 754  * Calculate the watermark level (the level at which the display plane will
 755  * start fetching from memory again).  Each chip has a different display
 756  * FIFO size and allocation, so the caller needs to figure that out and pass
 757  * in the correct intel_watermark_params structure.
 758  *
 759  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 760  * on the pixel size.  When it reaches the watermark level, it'll start
 761  * fetching FIFO line sized based chunks from memory until the FIFO fills
 762  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 763  * will occur, and a display engine hang could result.
 764  */
 765 static unsigned int intel_calculate_wm(int pixel_rate,
 766                                        const struct intel_watermark_params *wm,
 767                                        int fifo_size, int cpp,
 768                                        unsigned int latency_ns)
 769 {
 770         int entries, wm_size;
 771
 772         /*
 773          * Note: we need to make sure we don't overflow for various clock &
 774          * latency values.
 775          * clocks go from a few thousand to several hundred thousand.
 776          * latency is usually a few thousand
 777          */
 778         entries = intel_wm_method1(pixel_rate, cpp,
 779                                    latency_ns / 100);
 780         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 781                 wm->guard_size;
 782         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 783
 784         wm_size = fifo_size - entries;
 785         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 786
 787         /* Don't promote wm_size to unsigned... */
 788         if (wm_size > wm->max_wm)
 789                 wm_size = wm->max_wm;
 790         if (wm_size <= 0)
 791                 wm_size = wm->default_wm;
 792
 793         /*
 794          * Bspec seems to indicate that the value shouldn't be lower than
 795          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 796          * Lets go for 8 which is the burst size since certain platforms
 797          * already use a hardcoded 8 (which is what the spec says should be
 798          * done).
 799          */
 800         if (wm_size <= 8)
 801                 wm_size = 8;
 802
 803         return wm_size;
 804 }
 805
 806 static bool is_disabling(int old, int new, int threshold)
 807 {
 808         return old >= threshold && new < threshold;
 809 }
 810
 811 static bool is_enabling(int old, int new, int threshold)
 812 {
 813         return old < threshold && new >= threshold;
 814 }
 815
 816 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 817 {
 818         return dev_priv->wm.max_level + 1;
 819 }
 820
 821 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 822                                    const struct intel_plane_state *plane_state)
 823 {
 824         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 825
 826         /* FIXME check the 'enable' instead */
 827         if (!crtc_state->hw.active)
 828                 return false;
 829
 830         /*
 831          * Treat cursor with fb as always visible since cursor updates
 832          * can happen faster than the vrefresh rate, and the current
 833          * watermark code doesn't handle that correctly. Cursor updates
 834          * which set/clear the fb or change the cursor size are going
 835          * to get throttled by intel_legacy_cursor_update() to work
 836          * around this problem with the watermark code.
 837          */
 838         if (plane->id == PLANE_CURSOR)
 839                 return plane_state->hw.fb != NULL;
 840         else
 841                 return plane_state->uapi.visible;
 842 }
 843
 844 static bool intel_crtc_active(struct intel_crtc *crtc)
 845 {
 846         /* Be paranoid as we can arrive here with only partial
 847          * state retrieved from the hardware during setup.
 848          *
 849          * We can ditch the adjusted_mode.crtc_clock check as soon
 850          * as Haswell has gained clock readout/fastboot support.
 851          *
 852          * We can ditch the crtc->primary->state->fb check as soon as we can
 853          * properly reconstruct framebuffers.
 854          *
 855          * FIXME: The intel_crtc->active here should be switched to
 856          * crtc->state->active once we have proper CRTC states wired up
 857          * for atomic.
 858          */
 859         return crtc->active && crtc->base.primary->state->fb &&
 860                 crtc->config->hw.adjusted_mode.crtc_clock;
 861 }
 862
 863 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 864 {
 865         struct intel_crtc *crtc, *enabled = NULL;
 866
 867         for_each_intel_crtc(&dev_priv->drm, crtc) {
 868                 if (intel_crtc_active(crtc)) {
 869                         if (enabled)
 870                                 return NULL;
 871                         enabled = crtc;
 872                 }
 873         }
 874
 875         return enabled;
 876 }
 877
 878 static void pnv_update_wm(struct intel_crtc *unused_crtc)
 879 {
 880         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 881         struct intel_crtc *crtc;
 882         const struct cxsr_latency *latency;
 883         u32 reg;
 884         unsigned int wm;
 885
 886         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 887                                          dev_priv->is_ddr3,
 888                                          dev_priv->fsb_freq,
 889                                          dev_priv->mem_freq);
 890         if (!latency) {
 891                 drm_dbg_kms(&dev_priv->drm,
 892                             "Unknown FSB/MEM found, disable CxSR\n");
 893                 intel_set_memory_cxsr(dev_priv, false);
 894                 return;
 895         }
 896
 897         crtc = single_enabled_crtc(dev_priv);
 898         if (crtc) {
 899                 const struct drm_display_mode *adjusted_mode =
 900                         &crtc->config->hw.adjusted_mode;
 901                 const struct drm_framebuffer *fb =
 902                         crtc->base.primary->state->fb;
 903                 int cpp = fb->format->cpp[0];
 904                 int clock = adjusted_mode->crtc_clock;
 905
 906                 /* Display SR */
 907                 wm = intel_calculate_wm(clock, &pnv_display_wm,
 908                                         pnv_display_wm.fifo_size,
 909                                         cpp, latency->display_sr);
 910                 reg = I915_READ(DSPFW1);
 911                 reg &= ~DSPFW_SR_MASK;
 912                 reg |= FW_WM(wm, SR);
 913                 I915_WRITE(DSPFW1, reg);
 914                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 915
 916                 /* cursor SR */
 917                 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
 918                                         pnv_display_wm.fifo_size,
 919                                         4, latency->cursor_sr);
 920                 reg = I915_READ(DSPFW3);
 921                 reg &= ~DSPFW_CURSOR_SR_MASK;
 922                 reg |= FW_WM(wm, CURSOR_SR);
 923                 I915_WRITE(DSPFW3, reg);
 924
 925                 /* Display HPLL off SR */
 926                 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
 927                                         pnv_display_hplloff_wm.fifo_size,
 928                                         cpp, latency->display_hpll_disable);
 929                 reg = I915_READ(DSPFW3);
 930                 reg &= ~DSPFW_HPLL_SR_MASK;
 931                 reg |= FW_WM(wm, HPLL_SR);
 932                 I915_WRITE(DSPFW3, reg);
 933
 934                 /* cursor HPLL off SR */
 935                 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
 936                                         pnv_display_hplloff_wm.fifo_size,
 937                                         4, latency->cursor_hpll_disable);
 938                 reg = I915_READ(DSPFW3);
 939                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 940                 reg |= FW_WM(wm, HPLL_CURSOR);
 941                 I915_WRITE(DSPFW3, reg);
 942                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 943
 944                 intel_set_memory_cxsr(dev_priv, true);
 945         } else {
 946                 intel_set_memory_cxsr(dev_priv, false);
 947         }
 948 }
 949
 950 /*
 951  * Documentation says:
 952  * "If the line size is small, the TLB fetches can get in the way of the
 953  *  data fetches, causing some lag in the pixel data return which is not
 954  *  accounted for in the above formulas. The following adjustment only
 955  *  needs to be applied if eight whole lines fit in the buffer at once.
 956  *  The WM is adjusted upwards by the difference between the FIFO size
 957  *  and the size of 8 whole lines. This adjustment is always performed
 958  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 959  */
 960 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 961 {
 962         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 963
 964         return max(0, tlb_miss);
 965 }
 966
 967 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 968                                 const struct g4x_wm_values *wm)
 969 {
 970         enum pipe pipe;
 971
 972         for_each_pipe(dev_priv, pipe)
 973                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 974
 975         I915_WRITE(DSPFW1,
 976                    FW_WM(wm->sr.plane, SR) |
 977                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 978                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 979                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 980         I915_WRITE(DSPFW2,
 981                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 982                    FW_WM(wm->sr.fbc, FBC_SR) |
 983                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 984                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 985                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 986                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 987         I915_WRITE(DSPFW3,
 988                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 989                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 990                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 991                    FW_WM(wm->hpll.plane, HPLL_SR));
 992
 993         POSTING_READ(DSPFW1);
 994 }
 995
 996 #define FW_WM_VLV(value, plane) \
 997         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 998
 999 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1000                                 const struct vlv_wm_values *wm)
1001 {
1002         enum pipe pipe;
1003
1004         for_each_pipe(dev_priv, pipe) {
1005                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1006
1007                 I915_WRITE(VLV_DDL(pipe),
1008                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1009                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1010                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1011                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1012         }
1013
1014         /*
1015          * Zero the (unused) WM1 watermarks, and also clear all the
1016          * high order bits so that there are no out of bounds values
1017          * present in the registers during the reprogramming.
1018          */
1019         I915_WRITE(DSPHOWM, 0);
1020         I915_WRITE(DSPHOWM1, 0);
1021         I915_WRITE(DSPFW4, 0);
1022         I915_WRITE(DSPFW5, 0);
1023         I915_WRITE(DSPFW6, 0);
1024
1025         I915_WRITE(DSPFW1,
1026                    FW_WM(wm->sr.plane, SR) |
1027                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1028                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1029                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1030         I915_WRITE(DSPFW2,
1031                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1032                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1033                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1034         I915_WRITE(DSPFW3,
1035                    FW_WM(wm->sr.cursor, CURSOR_SR));
1036
1037         if (IS_CHERRYVIEW(dev_priv)) {
1038                 I915_WRITE(DSPFW7_CHV,
1039                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1040                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1041                 I915_WRITE(DSPFW8_CHV,
1042                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1043                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1044                 I915_WRITE(DSPFW9_CHV,
1045                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1046                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1047                 I915_WRITE(DSPHOWM,
1048                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1049                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1050                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1051                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1052                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1053                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1054                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1055                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1056                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1057                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1058         } else {
1059                 I915_WRITE(DSPFW7,
1060                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1061                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1062                 I915_WRITE(DSPHOWM,
1063                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1064                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1065                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1066                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1067                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1068                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1069                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1070         }
1071
1072         POSTING_READ(DSPFW1);
1073 }
1074
1075 #undef FW_WM_VLV
1076
1077 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1078 {
1079         /* all latencies in usec */
1080         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1081         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1082         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1083
1084         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1085 }
1086
1087 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1088 {
1089         /*
1090          * DSPCNTR[13] supposedly controls whether the
1091          * primary plane can use the FIFO space otherwise
1092          * reserved for the sprite plane. It's not 100% clear
1093          * what the actual FIFO size is, but it looks like we
1094          * can happily set both primary and sprite watermarks
1095          * up to 127 cachelines. So that would seem to mean
1096          * that either DSPCNTR[13] doesn't do anything, or that
1097          * the total FIFO is >= 256 cachelines in size. Either
1098          * way, we don't seem to have to worry about this
1099          * repartitioning as the maximum watermark value the
1100          * register can hold for each plane is lower than the
1101          * minimum FIFO size.
1102          */
1103         switch (plane_id) {
1104         case PLANE_CURSOR:
1105                 return 63;
1106         case PLANE_PRIMARY:
1107                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1108         case PLANE_SPRITE0:
1109                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1110         default:
1111                 MISSING_CASE(plane_id);
1112                 return 0;
1113         }
1114 }
1115
1116 static int g4x_fbc_fifo_size(int level)
1117 {
1118         switch (level) {
1119         case G4X_WM_LEVEL_SR:
1120                 return 7;
1121         case G4X_WM_LEVEL_HPLL:
1122                 return 15;
1123         default:
1124                 MISSING_CASE(level);
1125                 return 0;
1126         }
1127 }
1128
1129 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1130                           const struct intel_plane_state *plane_state,
1131                           int level)
1132 {
1133         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1134         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1135         const struct drm_display_mode *adjusted_mode =
1136                 &crtc_state->hw.adjusted_mode;
1137         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1138         unsigned int clock, htotal, cpp, width, wm;
1139
1140         if (latency == 0)
1141                 return USHRT_MAX;
1142
1143         if (!intel_wm_plane_visible(crtc_state, plane_state))
1144                 return 0;
1145
1146         cpp = plane_state->hw.fb->format->cpp[0];
1147
1148         /*
1149          * Not 100% sure which way ELK should go here as the
1150          * spec only says CL/CTG should assume 32bpp and BW
1151          * doesn't need to. But as these things followed the
1152          * mobile vs. desktop lines on gen3 as well, let's
1153          * assume ELK doesn't need this.
1154          *
1155          * The spec also fails to list such a restriction for
1156          * the HPLL watermark, which seems a little strange.
1157          * Let's use 32bpp for the HPLL watermark as well.
1158          */
1159         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1160             level != G4X_WM_LEVEL_NORMAL)
1161                 cpp = max(cpp, 4u);
1162
1163         clock = adjusted_mode->crtc_clock;
1164         htotal = adjusted_mode->crtc_htotal;
1165
1166         width = drm_rect_width(&plane_state->uapi.dst);
1167
1168         if (plane->id == PLANE_CURSOR) {
1169                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1170         } else if (plane->id == PLANE_PRIMARY &&
1171                    level == G4X_WM_LEVEL_NORMAL) {
1172                 wm = intel_wm_method1(clock, cpp, latency);
1173         } else {
1174                 unsigned int small, large;
1175
1176                 small = intel_wm_method1(clock, cpp, latency);
1177                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1178
1179                 wm = min(small, large);
1180         }
1181
1182         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1183                               width, cpp);
1184
1185         wm = DIV_ROUND_UP(wm, 64) + 2;
1186
1187         return min_t(unsigned int, wm, USHRT_MAX);
1188 }
1189
1190 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1191                                  int level, enum plane_id plane_id, u16 value)
1192 {
1193         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1194         bool dirty = false;
1195
1196         for (; level < intel_wm_num_levels(dev_priv); level++) {
1197                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1198
1199                 dirty |= raw->plane[plane_id] != value;
1200                 raw->plane[plane_id] = value;
1201         }
1202
1203         return dirty;
1204 }
1205
1206 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1207                                int level, u16 value)
1208 {
1209         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1210         bool dirty = false;
1211
1212         /* NORMAL level doesn't have an FBC watermark */
1213         level = max(level, G4X_WM_LEVEL_SR);
1214
1215         for (; level < intel_wm_num_levels(dev_priv); level++) {
1216                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1217
1218                 dirty |= raw->fbc != value;
1219                 raw->fbc = value;
1220         }
1221
1222         return dirty;
1223 }
1224
1225 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1226                               const struct intel_plane_state *plane_state,
1227                               u32 pri_val);
1228
1229 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1230                                      const struct intel_plane_state *plane_state)
1231 {
1232         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1233         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1234         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1235         enum plane_id plane_id = plane->id;
1236         bool dirty = false;
1237         int level;
1238
1239         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1240                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1241                 if (plane_id == PLANE_PRIMARY)
1242                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1243                 goto out;
1244         }
1245
1246         for (level = 0; level < num_levels; level++) {
1247                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1248                 int wm, max_wm;
1249
1250                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1251                 max_wm = g4x_plane_fifo_size(plane_id, level);
1252
1253                 if (wm > max_wm)
1254                         break;
1255
1256                 dirty |= raw->plane[plane_id] != wm;
1257                 raw->plane[plane_id] = wm;
1258
1259                 if (plane_id != PLANE_PRIMARY ||
1260                     level == G4X_WM_LEVEL_NORMAL)
1261                         continue;
1262
1263                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1264                                         raw->plane[plane_id]);
1265                 max_wm = g4x_fbc_fifo_size(level);
1266
1267                 /*
1268                  * FBC wm is not mandatory as we
1269                  * can always just disable its use.
1270                  */
1271                 if (wm > max_wm)
1272                         wm = USHRT_MAX;
1273
1274                 dirty |= raw->fbc != wm;
1275                 raw->fbc = wm;
1276         }
1277
1278         /* mark watermarks as invalid */
1279         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1280
1281         if (plane_id == PLANE_PRIMARY)
1282                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1283
1284  out:
1285         if (dirty) {
1286                 drm_dbg_kms(&dev_priv->drm,
1287                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1288                             plane->base.name,
1289                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1290                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1291                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1292
1293                 if (plane_id == PLANE_PRIMARY)
1294                         drm_dbg_kms(&dev_priv->drm,
1295                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1296                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1297                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1298         }
1299
1300         return dirty;
1301 }
1302
1303 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1304                                       enum plane_id plane_id, int level)
1305 {
1306         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1307
1308         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1309 }
1310
1311 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1312                                      int level)
1313 {
1314         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1315
1316         if (level > dev_priv->wm.max_level)
1317                 return false;
1318
1319         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1320                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1321                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1322 }
1323
1324 /* mark all levels starting from 'level' as invalid */
1325 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1326                                struct g4x_wm_state *wm_state, int level)
1327 {
1328         if (level <= G4X_WM_LEVEL_NORMAL) {
1329                 enum plane_id plane_id;
1330
1331                 for_each_plane_id_on_crtc(crtc, plane_id)
1332                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1333         }
1334
1335         if (level <= G4X_WM_LEVEL_SR) {
1336                 wm_state->cxsr = false;
1337                 wm_state->sr.cursor = USHRT_MAX;
1338                 wm_state->sr.plane = USHRT_MAX;
1339                 wm_state->sr.fbc = USHRT_MAX;
1340         }
1341
1342         if (level <= G4X_WM_LEVEL_HPLL) {
1343                 wm_state->hpll_en = false;
1344                 wm_state->hpll.cursor = USHRT_MAX;
1345                 wm_state->hpll.plane = USHRT_MAX;
1346                 wm_state->hpll.fbc = USHRT_MAX;
1347         }
1348 }
1349
1350 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1351                                int level)
1352 {
1353         if (level < G4X_WM_LEVEL_SR)
1354                 return false;
1355
1356         if (level >= G4X_WM_LEVEL_SR &&
1357             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1358                 return false;
1359
1360         if (level >= G4X_WM_LEVEL_HPLL &&
1361             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1362                 return false;
1363
1364         return true;
1365 }
1366
1367 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1368 {
1369         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1370         struct intel_atomic_state *state =
1371                 to_intel_atomic_state(crtc_state->uapi.state);
1372         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1373         int num_active_planes = hweight8(crtc_state->active_planes &
1374                                          ~BIT(PLANE_CURSOR));
1375         const struct g4x_pipe_wm *raw;
1376         const struct intel_plane_state *old_plane_state;
1377         const struct intel_plane_state *new_plane_state;
1378         struct intel_plane *plane;
1379         enum plane_id plane_id;
1380         int i, level;
1381         unsigned int dirty = 0;
1382
1383         for_each_oldnew_intel_plane_in_state(state, plane,
1384                                              old_plane_state,
1385                                              new_plane_state, i) {
1386                 if (new_plane_state->hw.crtc != &crtc->base &&
1387                     old_plane_state->hw.crtc != &crtc->base)
1388                         continue;
1389
1390                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1391                         dirty |= BIT(plane->id);
1392         }
1393
1394         if (!dirty)
1395                 return 0;
1396
1397         level = G4X_WM_LEVEL_NORMAL;
1398         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1399                 goto out;
1400
1401         raw = &crtc_state->wm.g4x.raw[level];
1402         for_each_plane_id_on_crtc(crtc, plane_id)
1403                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1404
1405         level = G4X_WM_LEVEL_SR;
1406         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1407                 goto out;
1408
1409         raw = &crtc_state->wm.g4x.raw[level];
1410         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1411         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1412         wm_state->sr.fbc = raw->fbc;
1413
1414         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1415
1416         level = G4X_WM_LEVEL_HPLL;
1417         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1418                 goto out;
1419
1420         raw = &crtc_state->wm.g4x.raw[level];
1421         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1422         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1423         wm_state->hpll.fbc = raw->fbc;
1424
1425         wm_state->hpll_en = wm_state->cxsr;
1426
1427         level++;
1428
1429  out:
1430         if (level == G4X_WM_LEVEL_NORMAL)
1431                 return -EINVAL;
1432
1433         /* invalidate the higher levels */
1434         g4x_invalidate_wms(crtc, wm_state, level);
1435
1436         /*
1437          * Determine if the FBC watermark(s) can be used. IF
1438          * this isn't the case we prefer to disable the FBC
1439          * watermark(s) rather than disable the SR/HPLL
1440          * level(s) entirely. 'level-1' is the highest valid
1441          * level here.
1442          */
1443         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1444
1445         return 0;
1446 }
1447
1448 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1449 {
1450         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1451         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1452         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1453         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1454         struct intel_atomic_state *intel_state =
1455                 to_intel_atomic_state(new_crtc_state->uapi.state);
1456         const struct intel_crtc_state *old_crtc_state =
1457                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1458         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1459         enum plane_id plane_id;
1460
1461         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1462                 *intermediate = *optimal;
1463
1464                 intermediate->cxsr = false;
1465                 intermediate->hpll_en = false;
1466                 goto out;
1467         }
1468
1469         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1470                 !new_crtc_state->disable_cxsr;
1471         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1472                 !new_crtc_state->disable_cxsr;
1473         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1474
1475         for_each_plane_id_on_crtc(crtc, plane_id) {
1476                 intermediate->wm.plane[plane_id] =
1477                         max(optimal->wm.plane[plane_id],
1478                             active->wm.plane[plane_id]);
1479
1480                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1481                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1482         }
1483
1484         intermediate->sr.plane = max(optimal->sr.plane,
1485                                      active->sr.plane);
1486         intermediate->sr.cursor = max(optimal->sr.cursor,
1487                                       active->sr.cursor);
1488         intermediate->sr.fbc = max(optimal->sr.fbc,
1489                                    active->sr.fbc);
1490
1491         intermediate->hpll.plane = max(optimal->hpll.plane,
1492                                        active->hpll.plane);
1493         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1494                                         active->hpll.cursor);
1495         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1496                                      active->hpll.fbc);
1497
1498         drm_WARN_ON(&dev_priv->drm,
1499                     (intermediate->sr.plane >
1500                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1501                      intermediate->sr.cursor >
1502                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1503                     intermediate->cxsr);
1504         drm_WARN_ON(&dev_priv->drm,
1505                     (intermediate->sr.plane >
1506                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1507                      intermediate->sr.cursor >
1508                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1509                     intermediate->hpll_en);
1510
1511         drm_WARN_ON(&dev_priv->drm,
1512                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1513                     intermediate->fbc_en && intermediate->cxsr);
1514         drm_WARN_ON(&dev_priv->drm,
1515                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1516                     intermediate->fbc_en && intermediate->hpll_en);
1517
1518 out:
1519         /*
1520          * If our intermediate WM are identical to the final WM, then we can
1521          * omit the post-vblank programming; only update if it's different.
1522          */
1523         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1524                 new_crtc_state->wm.need_postvbl_update = true;
1525
1526         return 0;
1527 }
1528
1529 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1530                          struct g4x_wm_values *wm)
1531 {
1532         struct intel_crtc *crtc;
1533         int num_active_pipes = 0;
1534
1535         wm->cxsr = true;
1536         wm->hpll_en = true;
1537         wm->fbc_en = true;
1538
1539         for_each_intel_crtc(&dev_priv->drm, crtc) {
1540                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1541
1542                 if (!crtc->active)
1543                         continue;
1544
1545                 if (!wm_state->cxsr)
1546                         wm->cxsr = false;
1547                 if (!wm_state->hpll_en)
1548                         wm->hpll_en = false;
1549                 if (!wm_state->fbc_en)
1550                         wm->fbc_en = false;
1551
1552                 num_active_pipes++;
1553         }
1554
1555         if (num_active_pipes != 1) {
1556                 wm->cxsr = false;
1557                 wm->hpll_en = false;
1558                 wm->fbc_en = false;
1559         }
1560
1561         for_each_intel_crtc(&dev_priv->drm, crtc) {
1562                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1563                 enum pipe pipe = crtc->pipe;
1564
1565                 wm->pipe[pipe] = wm_state->wm;
1566                 if (crtc->active && wm->cxsr)
1567                         wm->sr = wm_state->sr;
1568                 if (crtc->active && wm->hpll_en)
1569                         wm->hpll = wm_state->hpll;
1570         }
1571 }
1572
1573 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1574 {
1575         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1576         struct g4x_wm_values new_wm = {};
1577
1578         g4x_merge_wm(dev_priv, &new_wm);
1579
1580         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1581                 return;
1582
1583         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1584                 _intel_set_memory_cxsr(dev_priv, false);
1585
1586         g4x_write_wm_values(dev_priv, &new_wm);
1587
1588         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1589                 _intel_set_memory_cxsr(dev_priv, true);
1590
1591         *old_wm = new_wm;
1592 }
1593
1594 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1595                                    struct intel_crtc *crtc)
1596 {
1597         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1598         const struct intel_crtc_state *crtc_state =
1599                 intel_atomic_get_new_crtc_state(state, crtc);
1600
1601         mutex_lock(&dev_priv->wm.wm_mutex);
1602         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1603         g4x_program_watermarks(dev_priv);
1604         mutex_unlock(&dev_priv->wm.wm_mutex);
1605 }
1606
1607 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1608                                     struct intel_crtc *crtc)
1609 {
1610         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1611         const struct intel_crtc_state *crtc_state =
1612                 intel_atomic_get_new_crtc_state(state, crtc);
1613
1614         if (!crtc_state->wm.need_postvbl_update)
1615                 return;
1616
1617         mutex_lock(&dev_priv->wm.wm_mutex);
1618         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1619         g4x_program_watermarks(dev_priv);
1620         mutex_unlock(&dev_priv->wm.wm_mutex);
1621 }
1622
1623 /* latency must be in 0.1us units. */
1624 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1625                                    unsigned int htotal,
1626                                    unsigned int width,
1627                                    unsigned int cpp,
1628                                    unsigned int latency)
1629 {
1630         unsigned int ret;
1631
1632         ret = intel_wm_method2(pixel_rate, htotal,
1633                                width, cpp, latency);
1634         ret = DIV_ROUND_UP(ret, 64);
1635
1636         return ret;
1637 }
1638
1639 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1640 {
1641         /* all latencies in usec */
1642         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1643
1644         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1645
1646         if (IS_CHERRYVIEW(dev_priv)) {
1647                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1648                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1649
1650                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1651         }
1652 }
1653
1654 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1655                                 const struct intel_plane_state *plane_state,
1656                                 int level)
1657 {
1658         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1659         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1660         const struct drm_display_mode *adjusted_mode =
1661                 &crtc_state->hw.adjusted_mode;
1662         unsigned int clock, htotal, cpp, width, wm;
1663
1664         if (dev_priv->wm.pri_latency[level] == 0)
1665                 return USHRT_MAX;
1666
1667         if (!intel_wm_plane_visible(crtc_state, plane_state))
1668                 return 0;
1669
1670         cpp = plane_state->hw.fb->format->cpp[0];
1671         clock = adjusted_mode->crtc_clock;
1672         htotal = adjusted_mode->crtc_htotal;
1673         width = crtc_state->pipe_src_w;
1674
1675         if (plane->id == PLANE_CURSOR) {
1676                 /*
1677                  * FIXME the formula gives values that are
1678                  * too big for the cursor FIFO, and hence we
1679                  * would never be able to use cursors. For
1680                  * now just hardcode the watermark.
1681                  */
1682                 wm = 63;
1683         } else {
1684                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1685                                     dev_priv->wm.pri_latency[level] * 10);
1686         }
1687
1688         return min_t(unsigned int, wm, USHRT_MAX);
1689 }
1690
1691 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1692 {
1693         return (active_planes & (BIT(PLANE_SPRITE0) |
1694                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1695 }
1696
1697 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1698 {
1699         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1700         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1701         const struct g4x_pipe_wm *raw =
1702                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1703         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1704         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1705         int num_active_planes = hweight8(active_planes);
1706         const int fifo_size = 511;
1707         int fifo_extra, fifo_left = fifo_size;
1708         int sprite0_fifo_extra = 0;
1709         unsigned int total_rate;
1710         enum plane_id plane_id;
1711
1712         /*
1713          * When enabling sprite0 after sprite1 has already been enabled
1714          * we tend to get an underrun unless sprite0 already has some
1715          * FIFO space allcoated. Hence we always allocate at least one
1716          * cacheline for sprite0 whenever sprite1 is enabled.
1717          *
1718          * All other plane enable sequences appear immune to this problem.
1719          */
1720         if (vlv_need_sprite0_fifo_workaround(active_planes))
1721                 sprite0_fifo_extra = 1;
1722
1723         total_rate = raw->plane[PLANE_PRIMARY] +
1724                 raw->plane[PLANE_SPRITE0] +
1725                 raw->plane[PLANE_SPRITE1] +
1726                 sprite0_fifo_extra;
1727
1728         if (total_rate > fifo_size)
1729                 return -EINVAL;
1730
1731         if (total_rate == 0)
1732                 total_rate = 1;
1733
1734         for_each_plane_id_on_crtc(crtc, plane_id) {
1735                 unsigned int rate;
1736
1737                 if ((active_planes & BIT(plane_id)) == 0) {
1738                         fifo_state->plane[plane_id] = 0;
1739                         continue;
1740                 }
1741
1742                 rate = raw->plane[plane_id];
1743                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1744                 fifo_left -= fifo_state->plane[plane_id];
1745         }
1746
1747         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1748         fifo_left -= sprite0_fifo_extra;
1749
1750         fifo_state->plane[PLANE_CURSOR] = 63;
1751
1752         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1753
1754         /* spread the remainder evenly */
1755         for_each_plane_id_on_crtc(crtc, plane_id) {
1756                 int plane_extra;
1757
1758                 if (fifo_left == 0)
1759                         break;
1760
1761                 if ((active_planes & BIT(plane_id)) == 0)
1762                         continue;
1763
1764                 plane_extra = min(fifo_extra, fifo_left);
1765                 fifo_state->plane[plane_id] += plane_extra;
1766                 fifo_left -= plane_extra;
1767         }
1768
1769         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1770
1771         /* give it all to the first plane if none are active */
1772         if (active_planes == 0) {
1773                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1774                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1775         }
1776
1777         return 0;
1778 }
1779
1780 /* mark all levels starting from 'level' as invalid */
1781 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1782                                struct vlv_wm_state *wm_state, int level)
1783 {
1784         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1785
1786         for (; level < intel_wm_num_levels(dev_priv); level++) {
1787                 enum plane_id plane_id;
1788
1789                 for_each_plane_id_on_crtc(crtc, plane_id)
1790                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1791
1792                 wm_state->sr[level].cursor = USHRT_MAX;
1793                 wm_state->sr[level].plane = USHRT_MAX;
1794         }
1795 }
1796
1797 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1798 {
1799         if (wm > fifo_size)
1800                 return USHRT_MAX;
1801         else
1802                 return fifo_size - wm;
1803 }
1804
1805 /*
1806  * Starting from 'level' set all higher
1807  * levels to 'value' in the "raw" watermarks.
1808  */
1809 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1810                                  int level, enum plane_id plane_id, u16 value)
1811 {
1812         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1813         int num_levels = intel_wm_num_levels(dev_priv);
1814         bool dirty = false;
1815
1816         for (; level < num_levels; level++) {
1817                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1818
1819                 dirty |= raw->plane[plane_id] != value;
1820                 raw->plane[plane_id] = value;
1821         }
1822
1823         return dirty;
1824 }
1825
1826 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1827                                      const struct intel_plane_state *plane_state)
1828 {
1829         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1830         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1831         enum plane_id plane_id = plane->id;
1832         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1833         int level;
1834         bool dirty = false;
1835
1836         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1837                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1838                 goto out;
1839         }
1840
1841         for (level = 0; level < num_levels; level++) {
1842                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1843                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1844                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1845
1846                 if (wm > max_wm)
1847                         break;
1848
1849                 dirty |= raw->plane[plane_id] != wm;
1850                 raw->plane[plane_id] = wm;
1851         }
1852
1853         /* mark all higher levels as invalid */
1854         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1855
1856 out:
1857         if (dirty)
1858                 drm_dbg_kms(&dev_priv->drm,
1859                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1860                             plane->base.name,
1861                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1862                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1863                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1864
1865         return dirty;
1866 }
1867
1868 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1869                                       enum plane_id plane_id, int level)
1870 {
1871         const struct g4x_pipe_wm *raw =
1872                 &crtc_state->wm.vlv.raw[level];
1873         const struct vlv_fifo_state *fifo_state =
1874                 &crtc_state->wm.vlv.fifo_state;
1875
1876         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1877 }
1878
1879 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1880 {
1881         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1882                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1883                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1884                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1885 }
1886
1887 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1888 {
1889         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1890         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1891         struct intel_atomic_state *state =
1892                 to_intel_atomic_state(crtc_state->uapi.state);
1893         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1894         const struct vlv_fifo_state *fifo_state =
1895                 &crtc_state->wm.vlv.fifo_state;
1896         int num_active_planes = hweight8(crtc_state->active_planes &
1897                                          ~BIT(PLANE_CURSOR));
1898         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1899         const struct intel_plane_state *old_plane_state;
1900         const struct intel_plane_state *new_plane_state;
1901         struct intel_plane *plane;
1902         enum plane_id plane_id;
1903         int level, ret, i;
1904         unsigned int dirty = 0;
1905
1906         for_each_oldnew_intel_plane_in_state(state, plane,
1907                                              old_plane_state,
1908                                              new_plane_state, i) {
1909                 if (new_plane_state->hw.crtc != &crtc->base &&
1910                     old_plane_state->hw.crtc != &crtc->base)
1911                         continue;
1912
1913                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1914                         dirty |= BIT(plane->id);
1915         }
1916
1917         /*
1918          * DSPARB registers may have been reset due to the
1919          * power well being turned off. Make sure we restore
1920          * them to a consistent state even if no primary/sprite
1921          * planes are initially active.
1922          */
1923         if (needs_modeset)
1924                 crtc_state->fifo_changed = true;
1925
1926         if (!dirty)
1927                 return 0;
1928
1929         /* cursor changes don't warrant a FIFO recompute */
1930         if (dirty & ~BIT(PLANE_CURSOR)) {
1931                 const struct intel_crtc_state *old_crtc_state =
1932                         intel_atomic_get_old_crtc_state(state, crtc);
1933                 const struct vlv_fifo_state *old_fifo_state =
1934                         &old_crtc_state->wm.vlv.fifo_state;
1935
1936                 ret = vlv_compute_fifo(crtc_state);
1937                 if (ret)
1938                         return ret;
1939
1940                 if (needs_modeset ||
1941                     memcmp(old_fifo_state, fifo_state,
1942                            sizeof(*fifo_state)) != 0)
1943                         crtc_state->fifo_changed = true;
1944         }
1945
1946         /* initially allow all levels */
1947         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1948         /*
1949          * Note that enabling cxsr with no primary/sprite planes
1950          * enabled can wedge the pipe. Hence we only allow cxsr
1951          * with exactly one enabled primary/sprite plane.
1952          */
1953         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1954
1955         for (level = 0; level < wm_state->num_levels; level++) {
1956                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1957                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1958
1959                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1960                         break;
1961
1962                 for_each_plane_id_on_crtc(crtc, plane_id) {
1963                         wm_state->wm[level].plane[plane_id] =
1964                                 vlv_invert_wm_value(raw->plane[plane_id],
1965                                                     fifo_state->plane[plane_id]);
1966                 }
1967
1968                 wm_state->sr[level].plane =
1969                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1970                                                  raw->plane[PLANE_SPRITE0],
1971                                                  raw->plane[PLANE_SPRITE1]),
1972                                             sr_fifo_size);
1973
1974                 wm_state->sr[level].cursor =
1975                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1976                                             63);
1977         }
1978
1979         if (level == 0)
1980                 return -EINVAL;
1981
1982         /* limit to only levels we can actually handle */
1983         wm_state->num_levels = level;
1984
1985         /* invalidate the higher levels */
1986         vlv_invalidate_wms(crtc, wm_state, level);
1987
1988         return 0;
1989 }
1990
1991 #define VLV_FIFO(plane, value) \
1992         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1993
1994 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1995                                    struct intel_crtc *crtc)
1996 {
1997         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1998         struct intel_uncore *uncore = &dev_priv->uncore;
1999         const struct intel_crtc_state *crtc_state =
2000                 intel_atomic_get_new_crtc_state(state, crtc);
2001         const struct vlv_fifo_state *fifo_state =
2002                 &crtc_state->wm.vlv.fifo_state;
2003         int sprite0_start, sprite1_start, fifo_size;
2004         u32 dsparb, dsparb2, dsparb3;
2005
2006         if (!crtc_state->fifo_changed)
2007                 return;
2008
2009         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2010         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2011         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2012
2013         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2014         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2015
2016         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2017
2018         /*
2019          * uncore.lock serves a double purpose here. It allows us to
2020          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2021          * it protects the DSPARB registers from getting clobbered by
2022          * parallel updates from multiple pipes.
2023          *
2024          * intel_pipe_update_start() has already disabled interrupts
2025          * for us, so a plain spin_lock() is sufficient here.
2026          */
2027         spin_lock(&uncore->lock);
2028
2029         switch (crtc->pipe) {
2030         case PIPE_A:
2031                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2032                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2033
2034                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2035                             VLV_FIFO(SPRITEB, 0xff));
2036                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2037                            VLV_FIFO(SPRITEB, sprite1_start));
2038
2039                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2040                              VLV_FIFO(SPRITEB_HI, 0x1));
2041                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2042                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2043
2044                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2045                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2046                 break;
2047         case PIPE_B:
2048                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2049                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2050
2051                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2052                             VLV_FIFO(SPRITED, 0xff));
2053                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2054                            VLV_FIFO(SPRITED, sprite1_start));
2055
2056                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2057                              VLV_FIFO(SPRITED_HI, 0xff));
2058                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2059                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2060
2061                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2062                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2063                 break;
2064         case PIPE_C:
2065                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2066                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2067
2068                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2069                              VLV_FIFO(SPRITEF, 0xff));
2070                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2071                             VLV_FIFO(SPRITEF, sprite1_start));
2072
2073                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2074                              VLV_FIFO(SPRITEF_HI, 0xff));
2075                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2076                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2077
2078                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2079                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2080                 break;
2081         default:
2082                 break;
2083         }
2084
2085         intel_uncore_posting_read_fw(uncore, DSPARB);
2086
2087         spin_unlock(&uncore->lock);
2088 }
2089
2090 #undef VLV_FIFO
2091
2092 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2093 {
2094         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2095         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2096         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2097         struct intel_atomic_state *intel_state =
2098                 to_intel_atomic_state(new_crtc_state->uapi.state);
2099         const struct intel_crtc_state *old_crtc_state =
2100                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2101         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2102         int level;
2103
2104         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2105                 *intermediate = *optimal;
2106
2107                 intermediate->cxsr = false;
2108                 goto out;
2109         }
2110
2111         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2112         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2113                 !new_crtc_state->disable_cxsr;
2114
2115         for (level = 0; level < intermediate->num_levels; level++) {
2116                 enum plane_id plane_id;
2117
2118                 for_each_plane_id_on_crtc(crtc, plane_id) {
2119                         intermediate->wm[level].plane[plane_id] =
2120                                 min(optimal->wm[level].plane[plane_id],
2121                                     active->wm[level].plane[plane_id]);
2122                 }
2123
2124                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2125                                                     active->sr[level].plane);
2126                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2127                                                      active->sr[level].cursor);
2128         }
2129
2130         vlv_invalidate_wms(crtc, intermediate, level);
2131
2132 out:
2133         /*
2134          * If our intermediate WM are identical to the final WM, then we can
2135          * omit the post-vblank programming; only update if it's different.
2136          */
2137         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2138                 new_crtc_state->wm.need_postvbl_update = true;
2139
2140         return 0;
2141 }
2142
2143 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2144                          struct vlv_wm_values *wm)
2145 {
2146         struct intel_crtc *crtc;
2147         int num_active_pipes = 0;
2148
2149         wm->level = dev_priv->wm.max_level;
2150         wm->cxsr = true;
2151
2152         for_each_intel_crtc(&dev_priv->drm, crtc) {
2153                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2154
2155                 if (!crtc->active)
2156                         continue;
2157
2158                 if (!wm_state->cxsr)
2159                         wm->cxsr = false;
2160
2161                 num_active_pipes++;
2162                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2163         }
2164
2165         if (num_active_pipes != 1)
2166                 wm->cxsr = false;
2167
2168         if (num_active_pipes > 1)
2169                 wm->level = VLV_WM_LEVEL_PM2;
2170
2171         for_each_intel_crtc(&dev_priv->drm, crtc) {
2172                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2173                 enum pipe pipe = crtc->pipe;
2174
2175                 wm->pipe[pipe] = wm_state->wm[wm->level];
2176                 if (crtc->active && wm->cxsr)
2177                         wm->sr = wm_state->sr[wm->level];
2178
2179                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2180                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2181                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2182                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2183         }
2184 }
2185
2186 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2187 {
2188         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2189         struct vlv_wm_values new_wm = {};
2190
2191         vlv_merge_wm(dev_priv, &new_wm);
2192
2193         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2194                 return;
2195
2196         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2197                 chv_set_memory_dvfs(dev_priv, false);
2198
2199         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2200                 chv_set_memory_pm5(dev_priv, false);
2201
2202         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2203                 _intel_set_memory_cxsr(dev_priv, false);
2204
2205         vlv_write_wm_values(dev_priv, &new_wm);
2206
2207         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2208                 _intel_set_memory_cxsr(dev_priv, true);
2209
2210         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2211                 chv_set_memory_pm5(dev_priv, true);
2212
2213         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2214                 chv_set_memory_dvfs(dev_priv, true);
2215
2216         *old_wm = new_wm;
2217 }
2218
2219 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2220                                    struct intel_crtc *crtc)
2221 {
2222         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2223         const struct intel_crtc_state *crtc_state =
2224                 intel_atomic_get_new_crtc_state(state, crtc);
2225
2226         mutex_lock(&dev_priv->wm.wm_mutex);
2227         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2228         vlv_program_watermarks(dev_priv);
2229         mutex_unlock(&dev_priv->wm.wm_mutex);
2230 }
2231
2232 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2233                                     struct intel_crtc *crtc)
2234 {
2235         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2236         const struct intel_crtc_state *crtc_state =
2237                 intel_atomic_get_new_crtc_state(state, crtc);
2238
2239         if (!crtc_state->wm.need_postvbl_update)
2240                 return;
2241
2242         mutex_lock(&dev_priv->wm.wm_mutex);
2243         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2244         vlv_program_watermarks(dev_priv);
2245         mutex_unlock(&dev_priv->wm.wm_mutex);
2246 }
2247
2248 static void i965_update_wm(struct intel_crtc *unused_crtc)
2249 {
2250         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2251         struct intel_crtc *crtc;
2252         int srwm = 1;
2253         int cursor_sr = 16;
2254         bool cxsr_enabled;
2255
2256         /* Calc sr entries for one plane configs */
2257         crtc = single_enabled_crtc(dev_priv);
2258         if (crtc) {
2259                 /* self-refresh has much higher latency */
2260                 static const int sr_latency_ns = 12000;
2261                 const struct drm_display_mode *adjusted_mode =
2262                         &crtc->config->hw.adjusted_mode;
2263                 const struct drm_framebuffer *fb =
2264                         crtc->base.primary->state->fb;
2265                 int clock = adjusted_mode->crtc_clock;
2266                 int htotal = adjusted_mode->crtc_htotal;
2267                 int hdisplay = crtc->config->pipe_src_w;
2268                 int cpp = fb->format->cpp[0];
2269                 int entries;
2270
2271                 entries = intel_wm_method2(clock, htotal,
2272                                            hdisplay, cpp, sr_latency_ns / 100);
2273                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2274                 srwm = I965_FIFO_SIZE - entries;
2275                 if (srwm < 0)
2276                         srwm = 1;
2277                 srwm &= 0x1ff;
2278                 drm_dbg_kms(&dev_priv->drm,
2279                             "self-refresh entries: %d, wm: %d\n",
2280                             entries, srwm);
2281
2282                 entries = intel_wm_method2(clock, htotal,
2283                                            crtc->base.cursor->state->crtc_w, 4,
2284                                            sr_latency_ns / 100);
2285                 entries = DIV_ROUND_UP(entries,
2286                                        i965_cursor_wm_info.cacheline_size) +
2287                         i965_cursor_wm_info.guard_size;
2288
2289                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2290                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2291                         cursor_sr = i965_cursor_wm_info.max_wm;
2292
2293                 drm_dbg_kms(&dev_priv->drm,
2294                             "self-refresh watermark: display plane %d "
2295                             "cursor %d\n", srwm, cursor_sr);
2296
2297                 cxsr_enabled = true;
2298         } else {
2299                 cxsr_enabled = false;
2300                 /* Turn off self refresh if both pipes are enabled */
2301                 intel_set_memory_cxsr(dev_priv, false);
2302         }
2303
2304         drm_dbg_kms(&dev_priv->drm,
2305                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2306                     srwm);
2307
2308         /* 965 has limitations... */
2309         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2310                    FW_WM(8, CURSORB) |
2311                    FW_WM(8, PLANEB) |
2312                    FW_WM(8, PLANEA));
2313         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2314                    FW_WM(8, PLANEC_OLD));
2315         /* update cursor SR watermark */
2316         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2317
2318         if (cxsr_enabled)
2319                 intel_set_memory_cxsr(dev_priv, true);
2320 }
2321
2322 #undef FW_WM
2323
2324 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2325 {
2326         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2327         const struct intel_watermark_params *wm_info;
2328         u32 fwater_lo;
2329         u32 fwater_hi;
2330         int cwm, srwm = 1;
2331         int fifo_size;
2332         int planea_wm, planeb_wm;
2333         struct intel_crtc *crtc, *enabled = NULL;
2334
2335         if (IS_I945GM(dev_priv))
2336                 wm_info = &i945_wm_info;
2337         else if (!IS_GEN(dev_priv, 2))
2338                 wm_info = &i915_wm_info;
2339         else
2340                 wm_info = &i830_a_wm_info;
2341
2342         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2343         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2344         if (intel_crtc_active(crtc)) {
2345                 const struct drm_display_mode *adjusted_mode =
2346                         &crtc->config->hw.adjusted_mode;
2347                 const struct drm_framebuffer *fb =
2348                         crtc->base.primary->state->fb;
2349                 int cpp;
2350
2351                 if (IS_GEN(dev_priv, 2))
2352                         cpp = 4;
2353                 else
2354                         cpp = fb->format->cpp[0];
2355
2356                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2357                                                wm_info, fifo_size, cpp,
2358                                                pessimal_latency_ns);
2359                 enabled = crtc;
2360         } else {
2361                 planea_wm = fifo_size - wm_info->guard_size;
2362                 if (planea_wm > (long)wm_info->max_wm)
2363                         planea_wm = wm_info->max_wm;
2364         }
2365
2366         if (IS_GEN(dev_priv, 2))
2367                 wm_info = &i830_bc_wm_info;
2368
2369         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2370         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2371         if (intel_crtc_active(crtc)) {
2372                 const struct drm_display_mode *adjusted_mode =
2373                         &crtc->config->hw.adjusted_mode;
2374                 const struct drm_framebuffer *fb =
2375                         crtc->base.primary->state->fb;
2376                 int cpp;
2377
2378                 if (IS_GEN(dev_priv, 2))
2379                         cpp = 4;
2380                 else
2381                         cpp = fb->format->cpp[0];
2382
2383                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2384                                                wm_info, fifo_size, cpp,
2385                                                pessimal_latency_ns);
2386                 if (enabled == NULL)
2387                         enabled = crtc;
2388                 else
2389                         enabled = NULL;
2390         } else {
2391                 planeb_wm = fifo_size - wm_info->guard_size;
2392                 if (planeb_wm > (long)wm_info->max_wm)
2393                         planeb_wm = wm_info->max_wm;
2394         }
2395
2396         drm_dbg_kms(&dev_priv->drm,
2397                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2398
2399         if (IS_I915GM(dev_priv) && enabled) {
2400                 struct drm_i915_gem_object *obj;
2401
2402                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2403
2404                 /* self-refresh seems busted with untiled */
2405                 if (!i915_gem_object_is_tiled(obj))
2406                         enabled = NULL;
2407         }
2408
2409         /*
2410          * Overlay gets an aggressive default since video jitter is bad.
2411          */
2412         cwm = 2;
2413
2414         /* Play safe and disable self-refresh before adjusting watermarks. */
2415         intel_set_memory_cxsr(dev_priv, false);
2416
2417         /* Calc sr entries for one plane configs */
2418         if (HAS_FW_BLC(dev_priv) && enabled) {
2419                 /* self-refresh has much higher latency */
2420                 static const int sr_latency_ns = 6000;
2421                 const struct drm_display_mode *adjusted_mode =
2422                         &enabled->config->hw.adjusted_mode;
2423                 const struct drm_framebuffer *fb =
2424                         enabled->base.primary->state->fb;
2425                 int clock = adjusted_mode->crtc_clock;
2426                 int htotal = adjusted_mode->crtc_htotal;
2427                 int hdisplay = enabled->config->pipe_src_w;
2428                 int cpp;
2429                 int entries;
2430
2431                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2432                         cpp = 4;
2433                 else
2434                         cpp = fb->format->cpp[0];
2435
2436                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2437                                            sr_latency_ns / 100);
2438                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2439                 drm_dbg_kms(&dev_priv->drm,
2440                             "self-refresh entries: %d\n", entries);
2441                 srwm = wm_info->fifo_size - entries;
2442                 if (srwm < 0)
2443                         srwm = 1;
2444
2445                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2446                         I915_WRITE(FW_BLC_SELF,
2447                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2448                 else
2449                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2450         }
2451
2452         drm_dbg_kms(&dev_priv->drm,
2453                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2454                      planea_wm, planeb_wm, cwm, srwm);
2455
2456         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2457         fwater_hi = (cwm & 0x1f);
2458
2459         /* Set request length to 8 cachelines per fetch */
2460         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2461         fwater_hi = fwater_hi | (1 << 8);
2462
2463         I915_WRITE(FW_BLC, fwater_lo);
2464         I915_WRITE(FW_BLC2, fwater_hi);
2465
2466         if (enabled)
2467                 intel_set_memory_cxsr(dev_priv, true);
2468 }
2469
2470 static void i845_update_wm(struct intel_crtc *unused_crtc)
2471 {
2472         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2473         struct intel_crtc *crtc;
2474         const struct drm_display_mode *adjusted_mode;
2475         u32 fwater_lo;
2476         int planea_wm;
2477
2478         crtc = single_enabled_crtc(dev_priv);
2479         if (crtc == NULL)
2480                 return;
2481
2482         adjusted_mode = &crtc->config->hw.adjusted_mode;
2483         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2484                                        &i845_wm_info,
2485                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2486                                        4, pessimal_latency_ns);
2487         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2488         fwater_lo |= (3<<8) | planea_wm;
2489
2490         drm_dbg_kms(&dev_priv->drm,
2491                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2492
2493         I915_WRITE(FW_BLC, fwater_lo);
2494 }
2495
2496 /* latency must be in 0.1us units. */
2497 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2498                                    unsigned int cpp,
2499                                    unsigned int latency)
2500 {
2501         unsigned int ret;
2502
2503         ret = intel_wm_method1(pixel_rate, cpp, latency);
2504         ret = DIV_ROUND_UP(ret, 64) + 2;
2505
2506         return ret;
2507 }
2508
2509 /* latency must be in 0.1us units. */
2510 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2511                                    unsigned int htotal,
2512                                    unsigned int width,
2513                                    unsigned int cpp,
2514                                    unsigned int latency)
2515 {
2516         unsigned int ret;
2517
2518         ret = intel_wm_method2(pixel_rate, htotal,
2519                                width, cpp, latency);
2520         ret = DIV_ROUND_UP(ret, 64) + 2;
2521
2522         return ret;
2523 }
2524
2525 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2526 {
2527         /*
2528          * Neither of these should be possible since this function shouldn't be
2529          * called if the CRTC is off or the plane is invisible.  But let's be
2530          * extra paranoid to avoid a potential divide-by-zero if we screw up
2531          * elsewhere in the driver.
2532          */
2533         if (WARN_ON(!cpp))
2534                 return 0;
2535         if (WARN_ON(!horiz_pixels))
2536                 return 0;
2537
2538         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2539 }
2540
2541 struct ilk_wm_maximums {
2542         u16 pri;
2543         u16 spr;
2544         u16 cur;
2545         u16 fbc;
2546 };
2547
2548 /*
2549  * For both WM_PIPE and WM_LP.
2550  * mem_value must be in 0.1us units.
2551  */
2552 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2553                               const struct intel_plane_state *plane_state,
2554                               u32 mem_value, bool is_lp)
2555 {
2556         u32 method1, method2;
2557         int cpp;
2558
2559         if (mem_value == 0)
2560                 return U32_MAX;
2561
2562         if (!intel_wm_plane_visible(crtc_state, plane_state))
2563                 return 0;
2564
2565         cpp = plane_state->hw.fb->format->cpp[0];
2566
2567         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2568
2569         if (!is_lp)
2570                 return method1;
2571
2572         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2573                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2574                                  drm_rect_width(&plane_state->uapi.dst),
2575                                  cpp, mem_value);
2576
2577         return min(method1, method2);
2578 }
2579
2580 /*
2581  * For both WM_PIPE and WM_LP.
2582  * mem_value must be in 0.1us units.
2583  */
2584 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2585                               const struct intel_plane_state *plane_state,
2586                               u32 mem_value)
2587 {
2588         u32 method1, method2;
2589         int cpp;
2590
2591         if (mem_value == 0)
2592                 return U32_MAX;
2593
2594         if (!intel_wm_plane_visible(crtc_state, plane_state))
2595                 return 0;
2596
2597         cpp = plane_state->hw.fb->format->cpp[0];
2598
2599         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2600         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2601                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2602                                  drm_rect_width(&plane_state->uapi.dst),
2603                                  cpp, mem_value);
2604         return min(method1, method2);
2605 }
2606
2607 /*
2608  * For both WM_PIPE and WM_LP.
2609  * mem_value must be in 0.1us units.
2610  */
2611 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2612                               const struct intel_plane_state *plane_state,
2613                               u32 mem_value)
2614 {
2615         int cpp;
2616
2617         if (mem_value == 0)
2618                 return U32_MAX;
2619
2620         if (!intel_wm_plane_visible(crtc_state, plane_state))
2621                 return 0;
2622
2623         cpp = plane_state->hw.fb->format->cpp[0];
2624
2625         return ilk_wm_method2(crtc_state->pixel_rate,
2626                               crtc_state->hw.adjusted_mode.crtc_htotal,
2627                               drm_rect_width(&plane_state->uapi.dst),
2628                               cpp, mem_value);
2629 }
2630
2631 /* Only for WM_LP. */
2632 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2633                               const struct intel_plane_state *plane_state,
2634                               u32 pri_val)
2635 {
2636         int cpp;
2637
2638         if (!intel_wm_plane_visible(crtc_state, plane_state))
2639                 return 0;
2640
2641         cpp = plane_state->hw.fb->format->cpp[0];
2642
2643         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2644                           cpp);
2645 }
2646
2647 static unsigned int
2648 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2649 {
2650         if (INTEL_GEN(dev_priv) >= 8)
2651                 return 3072;
2652         else if (INTEL_GEN(dev_priv) >= 7)
2653                 return 768;
2654         else
2655                 return 512;
2656 }
2657
2658 static unsigned int
2659 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2660                      int level, bool is_sprite)
2661 {
2662         if (INTEL_GEN(dev_priv) >= 8)
2663                 /* BDW primary/sprite plane watermarks */
2664                 return level == 0 ? 255 : 2047;
2665         else if (INTEL_GEN(dev_priv) >= 7)
2666                 /* IVB/HSW primary/sprite plane watermarks */
2667                 return level == 0 ? 127 : 1023;
2668         else if (!is_sprite)
2669                 /* ILK/SNB primary plane watermarks */
2670                 return level == 0 ? 127 : 511;
2671         else
2672                 /* ILK/SNB sprite plane watermarks */
2673                 return level == 0 ? 63 : 255;
2674 }
2675
2676 static unsigned int
2677 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2678 {
2679         if (INTEL_GEN(dev_priv) >= 7)
2680                 return level == 0 ? 63 : 255;
2681         else
2682                 return level == 0 ? 31 : 63;
2683 }
2684
2685 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2686 {
2687         if (INTEL_GEN(dev_priv) >= 8)
2688                 return 31;
2689         else
2690                 return 15;
2691 }
2692
2693 /* Calculate the maximum primary/sprite plane watermark */
2694 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2695                                      int level,
2696                                      const struct intel_wm_config *config,
2697                                      enum intel_ddb_partitioning ddb_partitioning,
2698                                      bool is_sprite)
2699 {
2700         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2701
2702         /* if sprites aren't enabled, sprites get nothing */
2703         if (is_sprite && !config->sprites_enabled)
2704                 return 0;
2705
2706         /* HSW allows LP1+ watermarks even with multiple pipes */
2707         if (level == 0 || config->num_pipes_active > 1) {
2708                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2709
2710                 /*
2711                  * For some reason the non self refresh
2712                  * FIFO size is only half of the self
2713                  * refresh FIFO size on ILK/SNB.
2714                  */
2715                 if (INTEL_GEN(dev_priv) <= 6)
2716                         fifo_size /= 2;
2717         }
2718
2719         if (config->sprites_enabled) {
2720                 /* level 0 is always calculated with 1:1 split */
2721                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2722                         if (is_sprite)
2723                                 fifo_size *= 5;
2724                         fifo_size /= 6;
2725                 } else {
2726                         fifo_size /= 2;
2727                 }
2728         }
2729
2730         /* clamp to max that the registers can hold */
2731         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2732 }
2733
2734 /* Calculate the maximum cursor plane watermark */
2735 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2736                                       int level,
2737                                       const struct intel_wm_config *config)
2738 {
2739         /* HSW LP1+ watermarks w/ multiple pipes */
2740         if (level > 0 && config->num_pipes_active > 1)
2741                 return 64;
2742
2743         /* otherwise just report max that registers can hold */
2744         return ilk_cursor_wm_reg_max(dev_priv, level);
2745 }
2746
2747 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2748                                     int level,
2749                                     const struct intel_wm_config *config,
2750                                     enum intel_ddb_partitioning ddb_partitioning,
2751                                     struct ilk_wm_maximums *max)
2752 {
2753         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2754         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2755         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2756         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2757 }
2758
2759 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2760                                         int level,
2761                                         struct ilk_wm_maximums *max)
2762 {
2763         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2764         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2765         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2766         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2767 }
2768
2769 static bool ilk_validate_wm_level(int level,
2770                                   const struct ilk_wm_maximums *max,
2771                                   struct intel_wm_level *result)
2772 {
2773         bool ret;
2774
2775         /* already determined to be invalid? */
2776         if (!result->enable)
2777                 return false;
2778
2779         result->enable = result->pri_val <= max->pri &&
2780                          result->spr_val <= max->spr &&
2781                          result->cur_val <= max->cur;
2782
2783         ret = result->enable;
2784
2785         /*
2786          * HACK until we can pre-compute everything,
2787          * and thus fail gracefully if LP0 watermarks
2788          * are exceeded...
2789          */
2790         if (level == 0 && !result->enable) {
2791                 if (result->pri_val > max->pri)
2792                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2793                                       level, result->pri_val, max->pri);
2794                 if (result->spr_val > max->spr)
2795                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2796                                       level, result->spr_val, max->spr);
2797                 if (result->cur_val > max->cur)
2798                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2799                                       level, result->cur_val, max->cur);
2800
2801                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2802                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2803                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2804                 result->enable = true;
2805         }
2806
2807         return ret;
2808 }
2809
2810 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2811                                  const struct intel_crtc *crtc,
2812                                  int level,
2813                                  struct intel_crtc_state *crtc_state,
2814                                  const struct intel_plane_state *pristate,
2815                                  const struct intel_plane_state *sprstate,
2816                                  const struct intel_plane_state *curstate,
2817                                  struct intel_wm_level *result)
2818 {
2819         u16 pri_latency = dev_priv->wm.pri_latency[level];
2820         u16 spr_latency = dev_priv->wm.spr_latency[level];
2821         u16 cur_latency = dev_priv->wm.cur_latency[level];
2822
2823         /* WM1+ latency values stored in 0.5us units */
2824         if (level > 0) {
2825                 pri_latency *= 5;
2826                 spr_latency *= 5;
2827                 cur_latency *= 5;
2828         }
2829
2830         if (pristate) {
2831                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2832                                                      pri_latency, level);
2833                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2834         }
2835
2836         if (sprstate)
2837                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2838
2839         if (curstate)
2840                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2841
2842         result->enable = true;
2843 }
2844
2845 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2846                                   u16 wm[8])
2847 {
2848         struct intel_uncore *uncore = &dev_priv->uncore;
2849
2850         if (INTEL_GEN(dev_priv) >= 9) {
2851                 u32 val;
2852                 int ret, i;
2853                 int level, max_level = ilk_wm_max_level(dev_priv);
2854
2855                 /* read the first set of memory latencies[0:3] */
2856                 val = 0; /* data0 to be programmed to 0 for first set */
2857                 ret = sandybridge_pcode_read(dev_priv,
2858                                              GEN9_PCODE_READ_MEM_LATENCY,
2859                                              &val, NULL);
2860
2861                 if (ret) {
2862                         drm_err(&dev_priv->drm,
2863                                 "SKL Mailbox read error = %d\n", ret);
2864                         return;
2865                 }
2866
2867                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2868                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2869                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2870                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2871                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2872                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2873                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2874
2875                 /* read the second set of memory latencies[4:7] */
2876                 val = 1; /* data0 to be programmed to 1 for second set */
2877                 ret = sandybridge_pcode_read(dev_priv,
2878                                              GEN9_PCODE_READ_MEM_LATENCY,
2879                                              &val, NULL);
2880                 if (ret) {
2881                         drm_err(&dev_priv->drm,
2882                                 "SKL Mailbox read error = %d\n", ret);
2883                         return;
2884                 }
2885
2886                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2887                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2888                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2889                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2890                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2891                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2892                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2893
2894                 /*
2895                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2896                  * need to be disabled. We make sure to sanitize the values out
2897                  * of the punit to satisfy this requirement.
2898                  */
2899                 for (level = 1; level <= max_level; level++) {
2900                         if (wm[level] == 0) {
2901                                 for (i = level + 1; i <= max_level; i++)
2902                                         wm[i] = 0;
2903                                 break;
2904                         }
2905                 }
2906
2907                 /*
2908                  * WaWmMemoryReadLatency:skl+,glk
2909                  *
2910                  * punit doesn't take into account the read latency so we need
2911                  * to add 2us to the various latency levels we retrieve from the
2912                  * punit when level 0 response data us 0us.
2913                  */
2914                 if (wm[0] == 0) {
2915                         wm[0] += 2;
2916                         for (level = 1; level <= max_level; level++) {
2917                                 if (wm[level] == 0)
2918                                         break;
2919                                 wm[level] += 2;
2920                         }
2921                 }
2922
2923                 /*
2924                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2925                  * If we could not get dimm info enable this WA to prevent from
2926                  * any underrun. If not able to get Dimm info assume 16GB dimm
2927                  * to avoid any underrun.
2928                  */
2929                 if (dev_priv->dram_info.is_16gb_dimm)
2930                         wm[0] += 1;
2931
2932         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2933                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2934
2935                 wm[0] = (sskpd >> 56) & 0xFF;
2936                 if (wm[0] == 0)
2937                         wm[0] = sskpd & 0xF;
2938                 wm[1] = (sskpd >> 4) & 0xFF;
2939                 wm[2] = (sskpd >> 12) & 0xFF;
2940                 wm[3] = (sskpd >> 20) & 0x1FF;
2941                 wm[4] = (sskpd >> 32) & 0x1FF;
2942         } else if (INTEL_GEN(dev_priv) >= 6) {
2943                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2944
2945                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2946                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2947                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2948                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2949         } else if (INTEL_GEN(dev_priv) >= 5) {
2950                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2951
2952                 /* ILK primary LP0 latency is 700 ns */
2953                 wm[0] = 7;
2954                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2955                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2956         } else {
2957                 MISSING_CASE(INTEL_DEVID(dev_priv));
2958         }
2959 }
2960
2961 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2962                                        u16 wm[5])
2963 {
2964         /* ILK sprite LP0 latency is 1300 ns */
2965         if (IS_GEN(dev_priv, 5))
2966                 wm[0] = 13;
2967 }
2968
2969 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2970                                        u16 wm[5])
2971 {
2972         /* ILK cursor LP0 latency is 1300 ns */
2973         if (IS_GEN(dev_priv, 5))
2974                 wm[0] = 13;
2975 }
2976
2977 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2978 {
2979         /* how many WM levels are we expecting */
2980         if (INTEL_GEN(dev_priv) >= 9)
2981                 return 7;
2982         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2983                 return 4;
2984         else if (INTEL_GEN(dev_priv) >= 6)
2985                 return 3;
2986         else
2987                 return 2;
2988 }
2989
2990 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2991                                    const char *name,
2992                                    const u16 wm[8])
2993 {
2994         int level, max_level = ilk_wm_max_level(dev_priv);
2995
2996         for (level = 0; level <= max_level; level++) {
2997                 unsigned int latency = wm[level];
2998
2999                 if (latency == 0) {
3000                         drm_dbg_kms(&dev_priv->drm,
3001                                     "%s WM%d latency not provided\n",
3002                                     name, level);
3003                         continue;
3004                 }
3005
3006                 /*
3007                  * - latencies are in us on gen9.
3008                  * - before then, WM1+ latency values are in 0.5us units
3009                  */
3010                 if (INTEL_GEN(dev_priv) >= 9)
3011                         latency *= 10;
3012                 else if (level > 0)
3013                         latency *= 5;
3014
3015                 drm_dbg_kms(&dev_priv->drm,
3016                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
3017                             wm[level], latency / 10, latency % 10);
3018         }
3019 }
3020
3021 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3022                                     u16 wm[5], u16 min)
3023 {
3024         int level, max_level = ilk_wm_max_level(dev_priv);
3025
3026         if (wm[0] >= min)
3027                 return false;
3028
3029         wm[0] = max(wm[0], min);
3030         for (level = 1; level <= max_level; level++)
3031                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3032
3033         return true;
3034 }
3035
3036 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3037 {
3038         bool changed;
3039
3040         /*
3041          * The BIOS provided WM memory latency values are often
3042          * inadequate for high resolution displays. Adjust them.
3043          */
3044         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3045                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3046                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3047
3048         if (!changed)
3049                 return;
3050
3051         drm_dbg_kms(&dev_priv->drm,
3052                     "WM latency values increased to avoid potential underruns\n");
3053         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3054         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3055         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3056 }
3057
3058 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3059 {
3060         /*
3061          * On some SNB machines (Thinkpad X220 Tablet at least)
3062          * LP3 usage can cause vblank interrupts to be lost.
3063          * The DEIIR bit will go high but it looks like the CPU
3064          * never gets interrupted.
3065          *
3066          * It's not clear whether other interrupt source could
3067          * be affected or if this is somehow limited to vblank
3068          * interrupts only. To play it safe we disable LP3
3069          * watermarks entirely.
3070          */
3071         if (dev_priv->wm.pri_latency[3] == 0 &&
3072             dev_priv->wm.spr_latency[3] == 0 &&
3073             dev_priv->wm.cur_latency[3] == 0)
3074                 return;
3075
3076         dev_priv->wm.pri_latency[3] = 0;
3077         dev_priv->wm.spr_latency[3] = 0;
3078         dev_priv->wm.cur_latency[3] = 0;
3079
3080         drm_dbg_kms(&dev_priv->drm,
3081                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3082         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3083         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3084         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3085 }
3086
3087 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3088 {
3089         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3090
3091         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3092                sizeof(dev_priv->wm.pri_latency));
3093         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3094                sizeof(dev_priv->wm.pri_latency));
3095
3096         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3097         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3098
3099         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3100         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3101         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3102
3103         if (IS_GEN(dev_priv, 6)) {
3104                 snb_wm_latency_quirk(dev_priv);
3105                 snb_wm_lp3_irq_quirk(dev_priv);
3106         }
3107 }
3108
3109 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3110 {
3111         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3112         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3113 }
3114
3115 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3116                                  struct intel_pipe_wm *pipe_wm)
3117 {
3118         /* LP0 watermark maximums depend on this pipe alone */
3119         const struct intel_wm_config config = {
3120                 .num_pipes_active = 1,
3121                 .sprites_enabled = pipe_wm->sprites_enabled,
3122                 .sprites_scaled = pipe_wm->sprites_scaled,
3123         };
3124         struct ilk_wm_maximums max;
3125
3126         /* LP0 watermarks always use 1/2 DDB partitioning */
3127         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3128
3129         /* At least LP0 must be valid */
3130         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3131                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3132                 return false;
3133         }
3134
3135         return true;
3136 }
3137
3138 /* Compute new watermarks for the pipe */
3139 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3140 {
3141         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3142         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3143         struct intel_pipe_wm *pipe_wm;
3144         struct intel_plane *plane;
3145         const struct intel_plane_state *plane_state;
3146         const struct intel_plane_state *pristate = NULL;
3147         const struct intel_plane_state *sprstate = NULL;
3148         const struct intel_plane_state *curstate = NULL;
3149         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3150         struct ilk_wm_maximums max;
3151
3152         pipe_wm = &crtc_state->wm.ilk.optimal;
3153
3154         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3155                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3156                         pristate = plane_state;
3157                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3158                         sprstate = plane_state;
3159                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3160                         curstate = plane_state;
3161         }
3162
3163         pipe_wm->pipe_enabled = crtc_state->hw.active;
3164         if (sprstate) {
3165                 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3166                 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3167                         (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3168                          drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3169         }
3170
3171         usable_level = max_level;
3172
3173         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3174         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3175                 usable_level = 1;
3176
3177         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3178         if (pipe_wm->sprites_scaled)
3179                 usable_level = 0;
3180
3181         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3182         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3183                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3184
3185         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3186                 return -EINVAL;
3187
3188         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3189
3190         for (level = 1; level <= usable_level; level++) {
3191                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3192
3193                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3194                                      pristate, sprstate, curstate, wm);
3195
3196                 /*
3197                  * Disable any watermark level that exceeds the
3198                  * register maximums since such watermarks are
3199                  * always invalid.
3200                  */
3201                 if (!ilk_validate_wm_level(level, &max, wm)) {
3202                         memset(wm, 0, sizeof(*wm));
3203                         break;
3204                 }
3205         }
3206
3207         return 0;
3208 }
3209
3210 /*
3211  * Build a set of 'intermediate' watermark values that satisfy both the old
3212  * state and the new state.  These can be programmed to the hardware
3213  * immediately.
3214  */
3215 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3216 {
3217         struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3218         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3219         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3220         struct intel_atomic_state *intel_state =
3221                 to_intel_atomic_state(newstate->uapi.state);
3222         const struct intel_crtc_state *oldstate =
3223                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3224         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3225         int level, max_level = ilk_wm_max_level(dev_priv);
3226
3227         /*
3228          * Start with the final, target watermarks, then combine with the
3229          * currently active watermarks to get values that are safe both before
3230          * and after the vblank.
3231          */
3232         *a = newstate->wm.ilk.optimal;
3233         if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3234             intel_state->skip_intermediate_wm)
3235                 return 0;
3236
3237         a->pipe_enabled |= b->pipe_enabled;
3238         a->sprites_enabled |= b->sprites_enabled;
3239         a->sprites_scaled |= b->sprites_scaled;
3240
3241         for (level = 0; level <= max_level; level++) {
3242                 struct intel_wm_level *a_wm = &a->wm[level];
3243                 const struct intel_wm_level *b_wm = &b->wm[level];
3244
3245                 a_wm->enable &= b_wm->enable;
3246                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3247                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3248                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3249                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3250         }
3251
3252         /*
3253          * We need to make sure that these merged watermark values are
3254          * actually a valid configuration themselves.  If they're not,
3255          * there's no safe way to transition from the old state to
3256          * the new state, so we need to fail the atomic transaction.
3257          */
3258         if (!ilk_validate_pipe_wm(dev_priv, a))
3259                 return -EINVAL;
3260
3261         /*
3262          * If our intermediate WM are identical to the final WM, then we can
3263          * omit the post-vblank programming; only update if it's different.
3264          */
3265         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3266                 newstate->wm.need_postvbl_update = true;
3267
3268         return 0;
3269 }
3270
3271 /*
3272  * Merge the watermarks from all active pipes for a specific level.
3273  */
3274 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3275                                int level,
3276                                struct intel_wm_level *ret_wm)
3277 {
3278         const struct intel_crtc *intel_crtc;
3279
3280         ret_wm->enable = true;
3281
3282         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3283                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3284                 const struct intel_wm_level *wm = &active->wm[level];
3285
3286                 if (!active->pipe_enabled)
3287                         continue;
3288
3289                 /*
3290                  * The watermark values may have been used in the past,
3291                  * so we must maintain them in the registers for some
3292                  * time even if the level is now disabled.
3293                  */
3294                 if (!wm->enable)
3295                         ret_wm->enable = false;
3296
3297                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3298                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3299                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3300                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3301         }
3302 }
3303
3304 /*
3305  * Merge all low power watermarks for all active pipes.
3306  */
3307 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3308                          const struct intel_wm_config *config,
3309                          const struct ilk_wm_maximums *max,
3310                          struct intel_pipe_wm *merged)
3311 {
3312         int level, max_level = ilk_wm_max_level(dev_priv);
3313         int last_enabled_level = max_level;
3314
3315         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3316         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3317             config->num_pipes_active > 1)
3318                 last_enabled_level = 0;
3319
3320         /* ILK: FBC WM must be disabled always */
3321         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3322
3323         /* merge each WM1+ level */
3324         for (level = 1; level <= max_level; level++) {
3325                 struct intel_wm_level *wm = &merged->wm[level];
3326
3327                 ilk_merge_wm_level(dev_priv, level, wm);
3328
3329                 if (level > last_enabled_level)
3330                         wm->enable = false;
3331                 else if (!ilk_validate_wm_level(level, max, wm))
3332                         /* make sure all following levels get disabled */
3333                         last_enabled_level = level - 1;
3334
3335                 /*
3336                  * The spec says it is preferred to disable
3337                  * FBC WMs instead of disabling a WM level.
3338                  */
3339                 if (wm->fbc_val > max->fbc) {
3340                         if (wm->enable)
3341                                 merged->fbc_wm_enabled = false;
3342                         wm->fbc_val = 0;
3343                 }
3344         }
3345
3346         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3347         /*
3348          * FIXME this is racy. FBC might get enabled later.
3349          * What we should check here is whether FBC can be
3350          * enabled sometime later.
3351          */
3352         if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3353             intel_fbc_is_active(dev_priv)) {
3354                 for (level = 2; level <= max_level; level++) {
3355                         struct intel_wm_level *wm = &merged->wm[level];
3356
3357                         wm->enable = false;
3358                 }
3359         }
3360 }
3361
3362 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3363 {
3364         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3365         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3366 }
3367
3368 /* The value we need to program into the WM_LPx latency field */
3369 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3370                                       int level)
3371 {
3372         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3373                 return 2 * level;
3374         else
3375                 return dev_priv->wm.pri_latency[level];
3376 }
3377
3378 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3379                                    const struct intel_pipe_wm *merged,
3380                                    enum intel_ddb_partitioning partitioning,
3381                                    struct ilk_wm_values *results)
3382 {
3383         struct intel_crtc *intel_crtc;
3384         int level, wm_lp;
3385
3386         results->enable_fbc_wm = merged->fbc_wm_enabled;
3387         results->partitioning = partitioning;
3388
3389         /* LP1+ register values */
3390         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3391                 const struct intel_wm_level *r;
3392
3393                 level = ilk_wm_lp_to_level(wm_lp, merged);
3394
3395                 r = &merged->wm[level];
3396
3397                 /*
3398                  * Maintain the watermark values even if the level is
3399                  * disabled. Doing otherwise could cause underruns.
3400                  */
3401                 results->wm_lp[wm_lp - 1] =
3402                         (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3403                         (r->pri_val << WM1_LP_SR_SHIFT) |
3404                         r->cur_val;
3405
3406                 if (r->enable)
3407                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3408
3409                 if (INTEL_GEN(dev_priv) >= 8)
3410                         results->wm_lp[wm_lp - 1] |=
3411                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3412                 else
3413                         results->wm_lp[wm_lp - 1] |=
3414                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3415
3416                 /*
3417                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3418                  * level is disabled. Doing otherwise could cause underruns.
3419                  */
3420                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3421                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3422                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3423                 } else
3424                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3425         }
3426
3427         /* LP0 register values */
3428         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3429                 enum pipe pipe = intel_crtc->pipe;
3430                 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3431                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3432
3433                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3434                         continue;
3435
3436                 results->wm_pipe[pipe] =
3437                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3438                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3439                         r->cur_val;
3440         }
3441 }
3442
3443 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3444  * case both are at the same level. Prefer r1 in case they're the same. */
3445 static struct intel_pipe_wm *
3446 ilk_find_best_result(struct drm_i915_private *dev_priv,
3447                      struct intel_pipe_wm *r1,
3448                      struct intel_pipe_wm *r2)
3449 {
3450         int level, max_level = ilk_wm_max_level(dev_priv);
3451         int level1 = 0, level2 = 0;
3452
3453         for (level = 1; level <= max_level; level++) {
3454                 if (r1->wm[level].enable)
3455                         level1 = level;
3456                 if (r2->wm[level].enable)
3457                         level2 = level;
3458         }
3459
3460         if (level1 == level2) {
3461                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3462                         return r2;
3463                 else
3464                         return r1;
3465         } else if (level1 > level2) {
3466                 return r1;
3467         } else {
3468                 return r2;
3469         }
3470 }
3471
3472 /* dirty bits used to track which watermarks need changes */
3473 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3474 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3475 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3476 #define WM_DIRTY_FBC (1 << 24)
3477 #define WM_DIRTY_DDB (1 << 25)
3478
3479 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3480                                          const struct ilk_wm_values *old,
3481                                          const struct ilk_wm_values *new)
3482 {
3483         unsigned int dirty = 0;
3484         enum pipe pipe;
3485         int wm_lp;
3486
3487         for_each_pipe(dev_priv, pipe) {
3488                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3489                         dirty |= WM_DIRTY_PIPE(pipe);
3490                         /* Must disable LP1+ watermarks too */
3491                         dirty |= WM_DIRTY_LP_ALL;
3492                 }
3493         }
3494
3495         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3496                 dirty |= WM_DIRTY_FBC;
3497                 /* Must disable LP1+ watermarks too */
3498                 dirty |= WM_DIRTY_LP_ALL;
3499         }
3500
3501         if (old->partitioning != new->partitioning) {
3502                 dirty |= WM_DIRTY_DDB;
3503                 /* Must disable LP1+ watermarks too */
3504                 dirty |= WM_DIRTY_LP_ALL;
3505         }
3506
3507         /* LP1+ watermarks already deemed dirty, no need to continue */
3508         if (dirty & WM_DIRTY_LP_ALL)
3509                 return dirty;
3510
3511         /* Find the lowest numbered LP1+ watermark in need of an update... */
3512         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3513                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3514                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3515                         break;
3516         }
3517
3518         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3519         for (; wm_lp <= 3; wm_lp++)
3520                 dirty |= WM_DIRTY_LP(wm_lp);
3521
3522         return dirty;
3523 }
3524
3525 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3526                                unsigned int dirty)
3527 {
3528         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3529         bool changed = false;
3530
3531         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3532                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3533                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3534                 changed = true;
3535         }
3536         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3537                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3538                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3539                 changed = true;
3540         }
3541         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3542                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3543                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3544                 changed = true;
3545         }
3546
3547         /*
3548          * Don't touch WM1S_LP_EN here.
3549          * Doing so could cause underruns.
3550          */
3551
3552         return changed;
3553 }
3554
3555 /*
3556  * The spec says we shouldn't write when we don't need, because every write
3557  * causes WMs to be re-evaluated, expending some power.
3558  */
3559 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3560                                 struct ilk_wm_values *results)
3561 {
3562         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3563         unsigned int dirty;
3564         u32 val;
3565
3566         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3567         if (!dirty)
3568                 return;
3569
3570         _ilk_disable_lp_wm(dev_priv, dirty);
3571
3572         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3573                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3574         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3575                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3576         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3577                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3578
3579         if (dirty & WM_DIRTY_DDB) {
3580                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3581                         val = I915_READ(WM_MISC);
3582                         if (results->partitioning == INTEL_DDB_PART_1_2)
3583                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3584                         else
3585                                 val |= WM_MISC_DATA_PARTITION_5_6;
3586                         I915_WRITE(WM_MISC, val);
3587                 } else {
3588                         val = I915_READ(DISP_ARB_CTL2);
3589                         if (results->partitioning == INTEL_DDB_PART_1_2)
3590                                 val &= ~DISP_DATA_PARTITION_5_6;
3591                         else
3592                                 val |= DISP_DATA_PARTITION_5_6;
3593                         I915_WRITE(DISP_ARB_CTL2, val);
3594                 }
3595         }
3596
3597         if (dirty & WM_DIRTY_FBC) {
3598                 val = I915_READ(DISP_ARB_CTL);
3599                 if (results->enable_fbc_wm)
3600                         val &= ~DISP_FBC_WM_DIS;
3601                 else
3602                         val |= DISP_FBC_WM_DIS;
3603                 I915_WRITE(DISP_ARB_CTL, val);
3604         }
3605
3606         if (dirty & WM_DIRTY_LP(1) &&
3607             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3608                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3609
3610         if (INTEL_GEN(dev_priv) >= 7) {
3611                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3612                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3613                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3614                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3615         }
3616
3617         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3618                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3619         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3620                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3621         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3622                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3623
3624         dev_priv->wm.hw = *results;
3625 }
3626
3627 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3628 {
3629         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3630 }
3631
3632 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3633 {
3634         int i;
3635         int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3636         u8 enabled_slices_mask = 0;
3637
3638         for (i = 0; i < max_slices; i++) {
3639                 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3640                         enabled_slices_mask |= BIT(i);
3641         }
3642
3643         return enabled_slices_mask;
3644 }
3645
3646 /*
3647  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3648  * so assume we'll always need it in order to avoid underruns.
3649  */
3650 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3651 {
3652         return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3653 }
3654
3655 static bool
3656 intel_has_sagv(struct drm_i915_private *dev_priv)
3657 {
3658         return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3659                 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3660 }
3661
3662 static void
3663 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3664 {
3665         if (INTEL_GEN(dev_priv) >= 12) {
3666                 u32 val = 0;
3667                 int ret;
3668
3669                 ret = sandybridge_pcode_read(dev_priv,
3670                                              GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3671                                              &val, NULL);
3672                 if (!ret) {
3673                         dev_priv->sagv_block_time_us = val;
3674                         return;
3675                 }
3676
3677                 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3678         } else if (IS_GEN(dev_priv, 11)) {
3679                 dev_priv->sagv_block_time_us = 10;
3680                 return;
3681         } else if (IS_GEN(dev_priv, 10)) {
3682                 dev_priv->sagv_block_time_us = 20;
3683                 return;
3684         } else if (IS_GEN(dev_priv, 9)) {
3685                 dev_priv->sagv_block_time_us = 30;
3686                 return;
3687         } else {
3688                 MISSING_CASE(INTEL_GEN(dev_priv));
3689         }
3690
3691         /* Default to an unusable block time */
3692         dev_priv->sagv_block_time_us = -1;
3693 }
3694
3695 /*
3696  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3697  * depending on power and performance requirements. The display engine access
3698  * to system memory is blocked during the adjustment time. Because of the
3699  * blocking time, having this enabled can cause full system hangs and/or pipe
3700  * underruns if we don't meet all of the following requirements:
3701  *
3702  *  - <= 1 pipe enabled
3703  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3704  *  - We're not using an interlaced display configuration
3705  */
3706 int
3707 intel_enable_sagv(struct drm_i915_private *dev_priv)
3708 {
3709         int ret;
3710
3711         if (!intel_has_sagv(dev_priv))
3712                 return 0;
3713
3714         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3715                 return 0;
3716
3717         drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3718         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3719                                       GEN9_SAGV_ENABLE);
3720
3721         /* We don't need to wait for SAGV when enabling */
3722
3723         /*
3724          * Some skl systems, pre-release machines in particular,
3725          * don't actually have SAGV.
3726          */
3727         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3728                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3729                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3730                 return 0;
3731         } else if (ret < 0) {
3732                 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3733                 return ret;
3734         }
3735
3736         dev_priv->sagv_status = I915_SAGV_ENABLED;
3737         return 0;
3738 }
3739
3740 int
3741 intel_disable_sagv(struct drm_i915_private *dev_priv)
3742 {
3743         int ret;
3744
3745         if (!intel_has_sagv(dev_priv))
3746                 return 0;
3747
3748         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3749                 return 0;
3750
3751         drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3752         /* bspec says to keep retrying for at least 1 ms */
3753         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3754                                 GEN9_SAGV_DISABLE,
3755                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3756                                 1);
3757         /*
3758          * Some skl systems, pre-release machines in particular,
3759          * don't actually have SAGV.
3760          */
3761         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3762                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3763                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3764                 return 0;
3765         } else if (ret < 0) {
3766                 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3767                 return ret;
3768         }
3769
3770         dev_priv->sagv_status = I915_SAGV_DISABLED;
3771         return 0;
3772 }
3773
3774 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3775 {
3776         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3777         const struct intel_bw_state *new_bw_state;
3778         const struct intel_bw_state *old_bw_state;
3779         u32 new_mask = 0;
3780
3781         /*
3782          * Just return if we can't control SAGV or don't have it.
3783          * This is different from situation when we have SAGV but just can't
3784          * afford it due to DBuf limitation - in case if SAGV is completely
3785          * disabled in a BIOS, we are not even allowed to send a PCode request,
3786          * as it will throw an error. So have to check it here.
3787          */
3788         if (!intel_has_sagv(dev_priv))
3789                 return;
3790
3791         new_bw_state = intel_atomic_get_new_bw_state(state);
3792         if (!new_bw_state)
3793                 return;
3794
3795         if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3796                 intel_disable_sagv(dev_priv);
3797                 return;
3798         }
3799
3800         old_bw_state = intel_atomic_get_old_bw_state(state);
3801         /*
3802          * Nothing to mask
3803          */
3804         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3805                 return;
3806
3807         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3808
3809         /*
3810          * If new mask is zero - means there is nothing to mask,
3811          * we can only unmask, which should be done in unmask.
3812          */
3813         if (!new_mask)
3814                 return;
3815
3816         /*
3817          * Restrict required qgv points before updating the configuration.
3818          * According to BSpec we can't mask and unmask qgv points at the same
3819          * time. Also masking should be done before updating the configuration
3820          * and unmasking afterwards.
3821          */
3822         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3823 }
3824
3825 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3826 {
3827         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3828         const struct intel_bw_state *new_bw_state;
3829         const struct intel_bw_state *old_bw_state;
3830         u32 new_mask = 0;
3831
3832         /*
3833          * Just return if we can't control SAGV or don't have it.
3834          * This is different from situation when we have SAGV but just can't
3835          * afford it due to DBuf limitation - in case if SAGV is completely
3836          * disabled in a BIOS, we are not even allowed to send a PCode request,
3837          * as it will throw an error. So have to check it here.
3838          */
3839         if (!intel_has_sagv(dev_priv))
3840                 return;
3841
3842         new_bw_state = intel_atomic_get_new_bw_state(state);
3843         if (!new_bw_state)
3844                 return;
3845
3846         if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3847                 intel_enable_sagv(dev_priv);
3848                 return;
3849         }
3850
3851         old_bw_state = intel_atomic_get_old_bw_state(state);
3852         /*
3853          * Nothing to unmask
3854          */
3855         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3856                 return;
3857
3858         new_mask = new_bw_state->qgv_points_mask;
3859
3860         /*
3861          * Allow required qgv points after updating the configuration.
3862          * According to BSpec we can't mask and unmask qgv points at the same
3863          * time. Also masking should be done before updating the configuration
3864          * and unmasking afterwards.
3865          */
3866         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3867 }
3868
3869 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3870 {
3871         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3872         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3873         struct intel_plane *plane;
3874         const struct intel_plane_state *plane_state;
3875         int level, latency;
3876
3877         if (!intel_has_sagv(dev_priv))
3878                 return false;
3879
3880         if (!crtc_state->hw.active)
3881                 return true;
3882
3883         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3884                 return false;
3885
3886         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3887                 const struct skl_plane_wm *wm =
3888                         &crtc_state->wm.skl.optimal.planes[plane->id];
3889
3890                 /* Skip this plane if it's not enabled */
3891                 if (!wm->wm[0].plane_en)
3892                         continue;
3893
3894                 /* Find the highest enabled wm level for this plane */
3895                 for (level = ilk_wm_max_level(dev_priv);
3896                      !wm->wm[level].plane_en; --level)
3897                      { }
3898
3899                 latency = dev_priv->wm.skl_latency[level];
3900
3901                 if (skl_needs_memory_bw_wa(dev_priv) &&
3902                     plane_state->uapi.fb->modifier ==
3903                     I915_FORMAT_MOD_X_TILED)
3904                         latency += 15;
3905
3906                 /*
3907                  * If any of the planes on this pipe don't enable wm levels that
3908                  * incur memory latencies higher than sagv_block_time_us we
3909                  * can't enable SAGV.
3910                  */
3911                 if (latency < dev_priv->sagv_block_time_us)
3912                         return false;
3913         }
3914
3915         return true;
3916 }
3917
3918 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3919 {
3920         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3921         enum plane_id plane_id;
3922
3923         if (!crtc_state->hw.active)
3924                 return true;
3925
3926         for_each_plane_id_on_crtc(crtc, plane_id) {
3927                 const struct skl_ddb_entry *plane_alloc =
3928                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
3929                 const struct skl_plane_wm *wm =
3930                         &crtc_state->wm.skl.optimal.planes[plane_id];
3931
3932                 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3933                         return false;
3934         }
3935
3936         return true;
3937 }
3938
3939 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3940 {
3941         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3942         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3943
3944         if (INTEL_GEN(dev_priv) >= 12)
3945                 return tgl_crtc_can_enable_sagv(crtc_state);
3946         else
3947                 return skl_crtc_can_enable_sagv(crtc_state);
3948 }
3949
3950 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3951                            const struct intel_bw_state *bw_state)
3952 {
3953         if (INTEL_GEN(dev_priv) < 11 &&
3954             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3955                 return false;
3956
3957         return bw_state->pipe_sagv_reject == 0;
3958 }
3959
3960 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3961 {
3962         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3963         int ret;
3964         struct intel_crtc *crtc;
3965         struct intel_crtc_state *new_crtc_state;
3966         struct intel_bw_state *new_bw_state = NULL;
3967         const struct intel_bw_state *old_bw_state = NULL;
3968         int i;
3969
3970         for_each_new_intel_crtc_in_state(state, crtc,
3971                                          new_crtc_state, i) {
3972                 new_bw_state = intel_atomic_get_bw_state(state);
3973                 if (IS_ERR(new_bw_state))
3974                         return PTR_ERR(new_bw_state);
3975
3976                 old_bw_state = intel_atomic_get_old_bw_state(state);
3977
3978                 if (intel_crtc_can_enable_sagv(new_crtc_state))
3979                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3980                 else
3981                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3982         }
3983
3984         if (!new_bw_state)
3985                 return 0;
3986
3987         new_bw_state->active_pipes =
3988                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3989
3990         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3991                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3992                 if (ret)
3993                         return ret;
3994         }
3995
3996         for_each_new_intel_crtc_in_state(state, crtc,
3997                                          new_crtc_state, i) {
3998                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
3999
4000                 /*
4001                  * We store use_sagv_wm in the crtc state rather than relying on
4002                  * that bw state since we have no convenient way to get at the
4003                  * latter from the plane commit hooks (especially in the legacy
4004                  * cursor case)
4005                  */
4006                 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4007                                        intel_can_enable_sagv(dev_priv, new_bw_state);
4008         }
4009
4010         if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4011             intel_can_enable_sagv(dev_priv, old_bw_state)) {
4012                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4013                 if (ret)
4014                         return ret;
4015         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4016                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4017                 if (ret)
4018                         return ret;
4019         }
4020
4021         return 0;
4022 }
4023
4024 /*
4025  * Calculate initial DBuf slice offset, based on slice size
4026  * and mask(i.e if slice size is 1024 and second slice is enabled
4027  * offset would be 1024)
4028  */
4029 static unsigned int
4030 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4031                                 u32 slice_size,
4032                                 u32 ddb_size)
4033 {
4034         unsigned int offset = 0;
4035
4036         if (!dbuf_slice_mask)
4037                 return 0;
4038
4039         offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4040
4041         WARN_ON(offset >= ddb_size);
4042         return offset;
4043 }
4044
4045 u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4046 {
4047         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4048         drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4049
4050         if (INTEL_GEN(dev_priv) < 11)
4051                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4052
4053         return ddb_size;
4054 }
4055
4056 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4057                             const struct skl_ddb_entry *entry)
4058 {
4059         u32 slice_mask = 0;
4060         u16 ddb_size = intel_get_ddb_size(dev_priv);
4061         u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4062         u16 slice_size = ddb_size / num_supported_slices;
4063         u16 start_slice;
4064         u16 end_slice;
4065
4066         if (!skl_ddb_entry_size(entry))
4067                 return 0;
4068
4069         start_slice = entry->start / slice_size;
4070         end_slice = (entry->end - 1) / slice_size;
4071
4072         /*
4073          * Per plane DDB entry can in a really worst case be on multiple slices
4074          * but single entry is anyway contigious.
4075          */
4076         while (start_slice <= end_slice) {
4077                 slice_mask |= BIT(start_slice);
4078                 start_slice++;
4079         }
4080
4081         return slice_mask;
4082 }
4083
4084 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4085                                   u8 active_pipes);
4086
4087 static int
4088 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4089                                    const struct intel_crtc_state *crtc_state,
4090                                    const u64 total_data_rate,
4091                                    struct skl_ddb_entry *alloc, /* out */
4092                                    int *num_active /* out */)
4093 {
4094         struct drm_atomic_state *state = crtc_state->uapi.state;
4095         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4096         struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4097         const struct intel_crtc *crtc;
4098         u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4099         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4100         struct intel_dbuf_state *new_dbuf_state =
4101                 intel_atomic_get_new_dbuf_state(intel_state);
4102         const struct intel_dbuf_state *old_dbuf_state =
4103                 intel_atomic_get_old_dbuf_state(intel_state);
4104         u8 active_pipes = new_dbuf_state->active_pipes;
4105         u16 ddb_size;
4106         u32 ddb_range_size;
4107         u32 i;
4108         u32 dbuf_slice_mask;
4109         u32 offset;
4110         u32 slice_size;
4111         u32 total_slice_mask;
4112         u32 start, end;
4113         int ret;
4114
4115         *num_active = hweight8(active_pipes);
4116
4117         if (!crtc_state->hw.active) {
4118                 alloc->start = 0;
4119                 alloc->end = 0;
4120                 return 0;
4121         }
4122
4123         ddb_size = intel_get_ddb_size(dev_priv);
4124
4125         slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4126
4127         /*
4128          * If the state doesn't change the active CRTC's or there is no
4129          * modeset request, then there's no need to recalculate;
4130          * the existing pipe allocation limits should remain unchanged.
4131          * Note that we're safe from racing commits since any racing commit
4132          * that changes the active CRTC list or do modeset would need to
4133          * grab _all_ crtc locks, including the one we currently hold.
4134          */
4135         if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes &&
4136             !dev_priv->wm.distrust_bios_wm) {
4137                 /*
4138                  * alloc may be cleared by clear_intel_crtc_state,
4139                  * copy from old state to be sure
4140                  *
4141                  * FIXME get rid of this mess
4142                  */
4143                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4144                 return 0;
4145         }
4146
4147         /*
4148          * Get allowed DBuf slices for correspondent pipe and platform.
4149          */
4150         dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4151
4152         /*
4153          * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4154          * and slice size is 1024, the offset would be 1024
4155          */
4156         offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4157                                                  slice_size, ddb_size);
4158
4159         /*
4160          * Figure out total size of allowed DBuf slices, which is basically
4161          * a number of allowed slices for that pipe multiplied by slice size.
4162          * Inside of this
4163          * range ddb entries are still allocated in proportion to display width.
4164          */
4165         ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4166
4167         /*
4168          * Watermark/ddb requirement highly depends upon width of the
4169          * framebuffer, So instead of allocating DDB equally among pipes
4170          * distribute DDB based on resolution/width of the display.
4171          */
4172         total_slice_mask = dbuf_slice_mask;
4173         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4174                 const struct drm_display_mode *adjusted_mode =
4175                         &crtc_state->hw.adjusted_mode;
4176                 enum pipe pipe = crtc->pipe;
4177                 int hdisplay, vdisplay;
4178                 u32 pipe_dbuf_slice_mask;
4179
4180                 if (!crtc_state->hw.active)
4181                         continue;
4182
4183                 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4184                                                                active_pipes);
4185
4186                 /*
4187                  * According to BSpec pipe can share one dbuf slice with another
4188                  * pipes or pipe can use multiple dbufs, in both cases we
4189                  * account for other pipes only if they have exactly same mask.
4190                  * However we need to account how many slices we should enable
4191                  * in total.
4192                  */
4193                 total_slice_mask |= pipe_dbuf_slice_mask;
4194
4195                 /*
4196                  * Do not account pipes using other slice sets
4197                  * luckily as of current BSpec slice sets do not partially
4198                  * intersect(pipes share either same one slice or same slice set
4199                  * i.e no partial intersection), so it is enough to check for
4200                  * equality for now.
4201                  */
4202                 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4203                         continue;
4204
4205                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4206
4207                 total_width_in_range += hdisplay;
4208
4209                 if (pipe < for_pipe)
4210                         width_before_pipe_in_range += hdisplay;
4211                 else if (pipe == for_pipe)
4212                         pipe_width = hdisplay;
4213         }
4214
4215         /*
4216          * FIXME: For now we always enable slice S1 as per
4217          * the Bspec display initialization sequence.
4218          */
4219         new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1);
4220
4221         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4222                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4223                 if (ret)
4224                         return ret;
4225         }
4226
4227         start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4228         end = ddb_range_size *
4229                 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4230
4231         alloc->start = offset + start;
4232         alloc->end = offset + end;
4233
4234         drm_dbg_kms(&dev_priv->drm,
4235                     "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
4236                     for_crtc->base.id, for_crtc->name,
4237                     dbuf_slice_mask, alloc->start, alloc->end, active_pipes);
4238
4239         return 0;
4240 }
4241
4242 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4243                                  int width, const struct drm_format_info *format,
4244                                  u64 modifier, unsigned int rotation,
4245                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
4246                                  int color_plane);
4247 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4248                                  int level,
4249                                  unsigned int latency,
4250                                  const struct skl_wm_params *wp,
4251                                  const struct skl_wm_level *result_prev,
4252                                  struct skl_wm_level *result /* out */);
4253
4254 static unsigned int
4255 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4256                       int num_active)
4257 {
4258         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4259         int level, max_level = ilk_wm_max_level(dev_priv);
4260         struct skl_wm_level wm = {};
4261         int ret, min_ddb_alloc = 0;
4262         struct skl_wm_params wp;
4263
4264         ret = skl_compute_wm_params(crtc_state, 256,
4265                                     drm_format_info(DRM_FORMAT_ARGB8888),
4266                                     DRM_FORMAT_MOD_LINEAR,
4267                                     DRM_MODE_ROTATE_0,
4268                                     crtc_state->pixel_rate, &wp, 0);
4269         drm_WARN_ON(&dev_priv->drm, ret);
4270
4271         for (level = 0; level <= max_level; level++) {
4272                 unsigned int latency = dev_priv->wm.skl_latency[level];
4273
4274                 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4275                 if (wm.min_ddb_alloc == U16_MAX)
4276                         break;
4277
4278                 min_ddb_alloc = wm.min_ddb_alloc;
4279         }
4280
4281         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4282 }
4283
4284 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4285                                        struct skl_ddb_entry *entry, u32 reg)
4286 {
4287
4288         entry->start = reg & DDB_ENTRY_MASK;
4289         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4290
4291         if (entry->end)
4292                 entry->end += 1;
4293 }
4294
4295 static void
4296 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4297                            const enum pipe pipe,
4298                            const enum plane_id plane_id,
4299                            struct skl_ddb_entry *ddb_y,
4300                            struct skl_ddb_entry *ddb_uv)
4301 {
4302         u32 val, val2;
4303         u32 fourcc = 0;
4304
4305         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4306         if (plane_id == PLANE_CURSOR) {
4307                 val = I915_READ(CUR_BUF_CFG(pipe));
4308                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4309                 return;
4310         }
4311
4312         val = I915_READ(PLANE_CTL(pipe, plane_id));
4313
4314         /* No DDB allocated for disabled planes */
4315         if (val & PLANE_CTL_ENABLE)
4316                 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4317                                               val & PLANE_CTL_ORDER_RGBX,
4318                                               val & PLANE_CTL_ALPHA_MASK);
4319
4320         if (INTEL_GEN(dev_priv) >= 11) {
4321                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4322                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4323         } else {
4324                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4325                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4326
4327                 if (fourcc &&
4328                     drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4329                         swap(val, val2);
4330
4331                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4332                 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4333         }
4334 }
4335
4336 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4337                                struct skl_ddb_entry *ddb_y,
4338                                struct skl_ddb_entry *ddb_uv)
4339 {
4340         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4341         enum intel_display_power_domain power_domain;
4342         enum pipe pipe = crtc->pipe;
4343         intel_wakeref_t wakeref;
4344         enum plane_id plane_id;
4345
4346         power_domain = POWER_DOMAIN_PIPE(pipe);
4347         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4348         if (!wakeref)
4349                 return;
4350
4351         for_each_plane_id_on_crtc(crtc, plane_id)
4352                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4353                                            plane_id,
4354                                            &ddb_y[plane_id],
4355                                            &ddb_uv[plane_id]);
4356
4357         intel_display_power_put(dev_priv, power_domain, wakeref);
4358 }
4359
4360 /*
4361  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4362  * The bspec defines downscale amount as:
4363  *
4364  * """
4365  * Horizontal down scale amount = maximum[1, Horizontal source size /
4366  *                                           Horizontal destination size]
4367  * Vertical down scale amount = maximum[1, Vertical source size /
4368  *                                         Vertical destination size]
4369  * Total down scale amount = Horizontal down scale amount *
4370  *                           Vertical down scale amount
4371  * """
4372  *
4373  * Return value is provided in 16.16 fixed point form to retain fractional part.
4374  * Caller should take care of dividing & rounding off the value.
4375  */
4376 static uint_fixed_16_16_t
4377 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4378                            const struct intel_plane_state *plane_state)
4379 {
4380         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4381         u32 src_w, src_h, dst_w, dst_h;
4382         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4383         uint_fixed_16_16_t downscale_h, downscale_w;
4384
4385         if (drm_WARN_ON(&dev_priv->drm,
4386                         !intel_wm_plane_visible(crtc_state, plane_state)))
4387                 return u32_to_fixed16(0);
4388
4389         /*
4390          * Src coordinates are already rotated by 270 degrees for
4391          * the 90/270 degree plane rotation cases (to match the
4392          * GTT mapping), hence no need to account for rotation here.
4393          *
4394          * n.b., src is 16.16 fixed point, dst is whole integer.
4395          */
4396         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4397         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4398         dst_w = drm_rect_width(&plane_state->uapi.dst);
4399         dst_h = drm_rect_height(&plane_state->uapi.dst);
4400
4401         fp_w_ratio = div_fixed16(src_w, dst_w);
4402         fp_h_ratio = div_fixed16(src_h, dst_h);
4403         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4404         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4405
4406         return mul_fixed16(downscale_w, downscale_h);
4407 }
4408
4409 struct dbuf_slice_conf_entry {
4410         u8 active_pipes;
4411         u8 dbuf_mask[I915_MAX_PIPES];
4412 };
4413
4414 /*
4415  * Table taken from Bspec 12716
4416  * Pipes do have some preferred DBuf slice affinity,
4417  * plus there are some hardcoded requirements on how
4418  * those should be distributed for multipipe scenarios.
4419  * For more DBuf slices algorithm can get even more messy
4420  * and less readable, so decided to use a table almost
4421  * as is from BSpec itself - that way it is at least easier
4422  * to compare, change and check.
4423  */
4424 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4425 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4426 {
4427         {
4428                 .active_pipes = BIT(PIPE_A),
4429                 .dbuf_mask = {
4430                         [PIPE_A] = BIT(DBUF_S1),
4431                 },
4432         },
4433         {
4434                 .active_pipes = BIT(PIPE_B),
4435                 .dbuf_mask = {
4436                         [PIPE_B] = BIT(DBUF_S1),
4437                 },
4438         },
4439         {
4440                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4441                 .dbuf_mask = {
4442                         [PIPE_A] = BIT(DBUF_S1),
4443                         [PIPE_B] = BIT(DBUF_S2),
4444                 },
4445         },
4446         {
4447                 .active_pipes = BIT(PIPE_C),
4448                 .dbuf_mask = {
4449                         [PIPE_C] = BIT(DBUF_S2),
4450                 },
4451         },
4452         {
4453                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4454                 .dbuf_mask = {
4455                         [PIPE_A] = BIT(DBUF_S1),
4456                         [PIPE_C] = BIT(DBUF_S2),
4457                 },
4458         },
4459         {
4460                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4461                 .dbuf_mask = {
4462                         [PIPE_B] = BIT(DBUF_S1),
4463                         [PIPE_C] = BIT(DBUF_S2),
4464                 },
4465         },
4466         {
4467                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4468                 .dbuf_mask = {
4469                         [PIPE_A] = BIT(DBUF_S1),
4470                         [PIPE_B] = BIT(DBUF_S1),
4471                         [PIPE_C] = BIT(DBUF_S2),
4472                 },
4473         },
4474         {}
4475 };
4476
4477 /*
4478  * Table taken from Bspec 49255
4479  * Pipes do have some preferred DBuf slice affinity,
4480  * plus there are some hardcoded requirements on how
4481  * those should be distributed for multipipe scenarios.
4482  * For more DBuf slices algorithm can get even more messy
4483  * and less readable, so decided to use a table almost
4484  * as is from BSpec itself - that way it is at least easier
4485  * to compare, change and check.
4486  */
4487 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4488 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4489 {
4490         {
4491                 .active_pipes = BIT(PIPE_A),
4492                 .dbuf_mask = {
4493                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4494                 },
4495         },
4496         {
4497                 .active_pipes = BIT(PIPE_B),
4498                 .dbuf_mask = {
4499                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4500                 },
4501         },
4502         {
4503                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4504                 .dbuf_mask = {
4505                         [PIPE_A] = BIT(DBUF_S2),
4506                         [PIPE_B] = BIT(DBUF_S1),
4507                 },
4508         },
4509         {
4510                 .active_pipes = BIT(PIPE_C),
4511                 .dbuf_mask = {
4512                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4513                 },
4514         },
4515         {
4516                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4517                 .dbuf_mask = {
4518                         [PIPE_A] = BIT(DBUF_S1),
4519                         [PIPE_C] = BIT(DBUF_S2),
4520                 },
4521         },
4522         {
4523                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4524                 .dbuf_mask = {
4525                         [PIPE_B] = BIT(DBUF_S1),
4526                         [PIPE_C] = BIT(DBUF_S2),
4527                 },
4528         },
4529         {
4530                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4531                 .dbuf_mask = {
4532                         [PIPE_A] = BIT(DBUF_S1),
4533                         [PIPE_B] = BIT(DBUF_S1),
4534                         [PIPE_C] = BIT(DBUF_S2),
4535                 },
4536         },
4537         {
4538                 .active_pipes = BIT(PIPE_D),
4539                 .dbuf_mask = {
4540                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4541                 },
4542         },
4543         {
4544                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4545                 .dbuf_mask = {
4546                         [PIPE_A] = BIT(DBUF_S1),
4547                         [PIPE_D] = BIT(DBUF_S2),
4548                 },
4549         },
4550         {
4551                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4552                 .dbuf_mask = {
4553                         [PIPE_B] = BIT(DBUF_S1),
4554                         [PIPE_D] = BIT(DBUF_S2),
4555                 },
4556         },
4557         {
4558                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4559                 .dbuf_mask = {
4560                         [PIPE_A] = BIT(DBUF_S1),
4561                         [PIPE_B] = BIT(DBUF_S1),
4562                         [PIPE_D] = BIT(DBUF_S2),
4563                 },
4564         },
4565         {
4566                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4567                 .dbuf_mask = {
4568                         [PIPE_C] = BIT(DBUF_S1),
4569                         [PIPE_D] = BIT(DBUF_S2),
4570                 },
4571         },
4572         {
4573                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4574                 .dbuf_mask = {
4575                         [PIPE_A] = BIT(DBUF_S1),
4576                         [PIPE_C] = BIT(DBUF_S2),
4577                         [PIPE_D] = BIT(DBUF_S2),
4578                 },
4579         },
4580         {
4581                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4582                 .dbuf_mask = {
4583                         [PIPE_B] = BIT(DBUF_S1),
4584                         [PIPE_C] = BIT(DBUF_S2),
4585                         [PIPE_D] = BIT(DBUF_S2),
4586                 },
4587         },
4588         {
4589                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4590                 .dbuf_mask = {
4591                         [PIPE_A] = BIT(DBUF_S1),
4592                         [PIPE_B] = BIT(DBUF_S1),
4593                         [PIPE_C] = BIT(DBUF_S2),
4594                         [PIPE_D] = BIT(DBUF_S2),
4595                 },
4596         },
4597         {}
4598 };
4599
4600 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4601                               const struct dbuf_slice_conf_entry *dbuf_slices)
4602 {
4603         int i;
4604
4605         for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4606                 if (dbuf_slices[i].active_pipes == active_pipes)
4607                         return dbuf_slices[i].dbuf_mask[pipe];
4608         }
4609         return 0;
4610 }
4611
4612 /*
4613  * This function finds an entry with same enabled pipe configuration and
4614  * returns correspondent DBuf slice mask as stated in BSpec for particular
4615  * platform.
4616  */
4617 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4618 {
4619         /*
4620          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4621          * required calculating "pipe ratio" in order to determine
4622          * if one or two slices can be used for single pipe configurations
4623          * as additional constraint to the existing table.
4624          * However based on recent info, it should be not "pipe ratio"
4625          * but rather ratio between pixel_rate and cdclk with additional
4626          * constants, so for now we are using only table until this is
4627          * clarified. Also this is the reason why crtc_state param is
4628          * still here - we will need it once those additional constraints
4629          * pop up.
4630          */
4631         return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4632 }
4633
4634 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4635 {
4636         return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4637 }
4638
4639 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4640                                   u8 active_pipes)
4641 {
4642         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4643         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4644         enum pipe pipe = crtc->pipe;
4645
4646         if (IS_GEN(dev_priv, 12))
4647                 return tgl_compute_dbuf_slices(pipe, active_pipes);
4648         else if (IS_GEN(dev_priv, 11))
4649                 return icl_compute_dbuf_slices(pipe, active_pipes);
4650         /*
4651          * For anything else just return one slice yet.
4652          * Should be extended for other platforms.
4653          */
4654         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4655 }
4656
4657 static u64
4658 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4659                              const struct intel_plane_state *plane_state,
4660                              int color_plane)
4661 {
4662         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4663         const struct drm_framebuffer *fb = plane_state->hw.fb;
4664         u32 data_rate;
4665         u32 width = 0, height = 0;
4666         uint_fixed_16_16_t down_scale_amount;
4667         u64 rate;
4668
4669         if (!plane_state->uapi.visible)
4670                 return 0;
4671
4672         if (plane->id == PLANE_CURSOR)
4673                 return 0;
4674
4675         if (color_plane == 1 &&
4676             !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4677                 return 0;
4678
4679         /*
4680          * Src coordinates are already rotated by 270 degrees for
4681          * the 90/270 degree plane rotation cases (to match the
4682          * GTT mapping), hence no need to account for rotation here.
4683          */
4684         width = drm_rect_width(&plane_state->uapi.src) >> 16;
4685         height = drm_rect_height(&plane_state->uapi.src) >> 16;
4686
4687         /* UV plane does 1/2 pixel sub-sampling */
4688         if (color_plane == 1) {
4689                 width /= 2;
4690                 height /= 2;
4691         }
4692
4693         data_rate = width * height;
4694
4695         down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4696
4697         rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4698
4699         rate *= fb->format->cpp[color_plane];
4700         return rate;
4701 }
4702
4703 static u64
4704 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4705                                  u64 *plane_data_rate,
4706                                  u64 *uv_plane_data_rate)
4707 {
4708         struct intel_plane *plane;
4709         const struct intel_plane_state *plane_state;
4710         u64 total_data_rate = 0;
4711
4712         /* Calculate and cache data rate for each plane */
4713         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4714                 enum plane_id plane_id = plane->id;
4715                 u64 rate;
4716
4717                 /* packed/y */
4718                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4719                 plane_data_rate[plane_id] = rate;
4720                 total_data_rate += rate;
4721
4722                 /* uv-plane */
4723                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4724                 uv_plane_data_rate[plane_id] = rate;
4725                 total_data_rate += rate;
4726         }
4727
4728         return total_data_rate;
4729 }
4730
4731 static u64
4732 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4733                                  u64 *plane_data_rate)
4734 {
4735         struct intel_plane *plane;
4736         const struct intel_plane_state *plane_state;
4737         u64 total_data_rate = 0;
4738
4739         /* Calculate and cache data rate for each plane */
4740         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4741                 enum plane_id plane_id = plane->id;
4742                 u64 rate;
4743
4744                 if (!plane_state->planar_linked_plane) {
4745                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4746                         plane_data_rate[plane_id] = rate;
4747                         total_data_rate += rate;
4748                 } else {
4749                         enum plane_id y_plane_id;
4750
4751                         /*
4752                          * The slave plane might not iterate in
4753                          * intel_atomic_crtc_state_for_each_plane_state(),
4754                          * and needs the master plane state which may be
4755                          * NULL if we try get_new_plane_state(), so we
4756                          * always calculate from the master.
4757                          */
4758                         if (plane_state->planar_slave)
4759                                 continue;
4760
4761                         /* Y plane rate is calculated on the slave */
4762                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4763                         y_plane_id = plane_state->planar_linked_plane->id;
4764                         plane_data_rate[y_plane_id] = rate;
4765                         total_data_rate += rate;
4766
4767                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4768                         plane_data_rate[plane_id] = rate;
4769                         total_data_rate += rate;
4770                 }
4771         }
4772
4773         return total_data_rate;
4774 }
4775
4776 static const struct skl_wm_level *
4777 skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4778                    enum plane_id plane_id,
4779                    int level)
4780 {
4781         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4782         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4783
4784         if (level == 0 && pipe_wm->use_sagv_wm)
4785                 return &wm->sagv_wm0;
4786
4787         return &wm->wm[level];
4788 }
4789
4790 static int
4791 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4792 {
4793         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4794         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4795         struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4796         u16 alloc_size, start = 0;
4797         u16 total[I915_MAX_PLANES] = {};
4798         u16 uv_total[I915_MAX_PLANES] = {};
4799         u64 total_data_rate;
4800         enum plane_id plane_id;
4801         int num_active;
4802         u64 plane_data_rate[I915_MAX_PLANES] = {};
4803         u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4804         u32 blocks;
4805         int level;
4806         int ret;
4807
4808         /* Clear the partitioning for disabled planes. */
4809         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4810         memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4811
4812         if (!crtc_state->hw.active) {
4813                 struct intel_atomic_state *state =
4814                         to_intel_atomic_state(crtc_state->uapi.state);
4815                 struct intel_dbuf_state *new_dbuf_state =
4816                         intel_atomic_get_new_dbuf_state(state);
4817                 const struct intel_dbuf_state *old_dbuf_state =
4818                         intel_atomic_get_old_dbuf_state(state);
4819
4820                 /*
4821                  * FIXME hack to make sure we compute this sensibly when
4822                  * turning off all the pipes. Otherwise we leave it at
4823                  * whatever we had previously, and then runtime PM will
4824                  * mess it up by turning off all but S1. Remove this
4825                  * once the dbuf state computation flow becomes sane.
4826                  */
4827                 if (new_dbuf_state->active_pipes == 0) {
4828                         new_dbuf_state->enabled_slices = BIT(DBUF_S1);
4829
4830                         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4831                                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4832                                 if (ret)
4833                                         return ret;
4834                         }
4835                 }
4836
4837                 alloc->start = alloc->end = 0;
4838                 return 0;
4839         }
4840
4841         if (INTEL_GEN(dev_priv) >= 11)
4842                 total_data_rate =
4843                         icl_get_total_relative_data_rate(crtc_state,
4844                                                          plane_data_rate);
4845         else
4846                 total_data_rate =
4847                         skl_get_total_relative_data_rate(crtc_state,
4848                                                          plane_data_rate,
4849                                                          uv_plane_data_rate);
4850
4851         ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state,
4852                                                  total_data_rate,
4853                                                  alloc, &num_active);
4854         if (ret)
4855                 return ret;
4856
4857         alloc_size = skl_ddb_entry_size(alloc);
4858         if (alloc_size == 0)
4859                 return 0;
4860
4861         /* Allocate fixed number of blocks for cursor. */
4862         total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4863         alloc_size -= total[PLANE_CURSOR];
4864         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4865                 alloc->end - total[PLANE_CURSOR];
4866         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4867
4868         if (total_data_rate == 0)
4869                 return 0;
4870
4871         /*
4872          * Find the highest watermark level for which we can satisfy the block
4873          * requirement of active planes.
4874          */
4875         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4876                 blocks = 0;
4877                 for_each_plane_id_on_crtc(crtc, plane_id) {
4878                         const struct skl_plane_wm *wm =
4879                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4880
4881                         if (plane_id == PLANE_CURSOR) {
4882                                 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4883                                         drm_WARN_ON(&dev_priv->drm,
4884                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
4885                                         blocks = U32_MAX;
4886                                         break;
4887                                 }
4888                                 continue;
4889                         }
4890
4891                         blocks += wm->wm[level].min_ddb_alloc;
4892                         blocks += wm->uv_wm[level].min_ddb_alloc;
4893                 }
4894
4895                 if (blocks <= alloc_size) {
4896                         alloc_size -= blocks;
4897                         break;
4898                 }
4899         }
4900
4901         if (level < 0) {
4902                 drm_dbg_kms(&dev_priv->drm,
4903                             "Requested display configuration exceeds system DDB limitations");
4904                 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4905                             blocks, alloc_size);
4906                 return -EINVAL;
4907         }
4908
4909         /*
4910          * Grant each plane the blocks it requires at the highest achievable
4911          * watermark level, plus an extra share of the leftover blocks
4912          * proportional to its relative data rate.
4913          */
4914         for_each_plane_id_on_crtc(crtc, plane_id) {
4915                 const struct skl_plane_wm *wm =
4916                         &crtc_state->wm.skl.optimal.planes[plane_id];
4917                 u64 rate;
4918                 u16 extra;
4919
4920                 if (plane_id == PLANE_CURSOR)
4921                         continue;
4922
4923                 /*
4924                  * We've accounted for all active planes; remaining planes are
4925                  * all disabled.
4926                  */
4927                 if (total_data_rate == 0)
4928                         break;
4929
4930                 rate = plane_data_rate[plane_id];
4931                 extra = min_t(u16, alloc_size,
4932                               DIV64_U64_ROUND_UP(alloc_size * rate,
4933                                                  total_data_rate));
4934                 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4935                 alloc_size -= extra;
4936                 total_data_rate -= rate;
4937
4938                 if (total_data_rate == 0)
4939                         break;
4940
4941                 rate = uv_plane_data_rate[plane_id];
4942                 extra = min_t(u16, alloc_size,
4943                               DIV64_U64_ROUND_UP(alloc_size * rate,
4944                                                  total_data_rate));
4945                 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4946                 alloc_size -= extra;
4947                 total_data_rate -= rate;
4948         }
4949         drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4950
4951         /* Set the actual DDB start/end points for each plane */
4952         start = alloc->start;
4953         for_each_plane_id_on_crtc(crtc, plane_id) {
4954                 struct skl_ddb_entry *plane_alloc =
4955                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
4956                 struct skl_ddb_entry *uv_plane_alloc =
4957                         &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4958
4959                 if (plane_id == PLANE_CURSOR)
4960                         continue;
4961
4962                 /* Gen11+ uses a separate plane for UV watermarks */
4963                 drm_WARN_ON(&dev_priv->drm,
4964                             INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4965
4966                 /* Leave disabled planes at (0,0) */
4967                 if (total[plane_id]) {
4968                         plane_alloc->start = start;
4969                         start += total[plane_id];
4970                         plane_alloc->end = start;
4971                 }
4972
4973                 if (uv_total[plane_id]) {
4974                         uv_plane_alloc->start = start;
4975                         start += uv_total[plane_id];
4976                         uv_plane_alloc->end = start;
4977                 }
4978         }
4979
4980         /*
4981          * When we calculated watermark values we didn't know how high
4982          * of a level we'd actually be able to hit, so we just marked
4983          * all levels as "enabled."  Go back now and disable the ones
4984          * that aren't actually possible.
4985          */
4986         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4987                 for_each_plane_id_on_crtc(crtc, plane_id) {
4988                         struct skl_plane_wm *wm =
4989                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4990
4991                         /*
4992                          * We only disable the watermarks for each plane if
4993                          * they exceed the ddb allocation of said plane. This
4994                          * is done so that we don't end up touching cursor
4995                          * watermarks needlessly when some other plane reduces
4996                          * our max possible watermark level.
4997                          *
4998                          * Bspec has this to say about the PLANE_WM enable bit:
4999                          * "All the watermarks at this level for all enabled
5000                          *  planes must be enabled before the level will be used."
5001                          * So this is actually safe to do.
5002                          */
5003                         if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
5004                             wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
5005                                 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
5006
5007                         /*
5008                          * Wa_1408961008:icl, ehl
5009                          * Underruns with WM1+ disabled
5010                          */
5011                         if (IS_GEN(dev_priv, 11) &&
5012                             level == 1 && wm->wm[0].plane_en) {
5013                                 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
5014                                 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
5015                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5016                         }
5017                 }
5018         }
5019
5020         /*
5021          * Go back and disable the transition watermark if it turns out we
5022          * don't have enough DDB blocks for it.
5023          */
5024         for_each_plane_id_on_crtc(crtc, plane_id) {
5025                 struct skl_plane_wm *wm =
5026                         &crtc_state->wm.skl.optimal.planes[plane_id];
5027
5028                 if (wm->trans_wm.plane_res_b >= total[plane_id])
5029                         memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
5030         }
5031
5032         return 0;
5033 }
5034
5035 /*
5036  * The max latency should be 257 (max the punit can code is 255 and we add 2us
5037  * for the read latency) and cpp should always be <= 8, so that
5038  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5039  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5040 */
5041 static uint_fixed_16_16_t
5042 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5043                u8 cpp, u32 latency, u32 dbuf_block_size)
5044 {
5045         u32 wm_intermediate_val;
5046         uint_fixed_16_16_t ret;
5047
5048         if (latency == 0)
5049                 return FP_16_16_MAX;
5050
5051         wm_intermediate_val = latency * pixel_rate * cpp;
5052         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5053
5054         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5055                 ret = add_fixed16_u32(ret, 1);
5056
5057         return ret;
5058 }
5059
5060 static uint_fixed_16_16_t
5061 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5062                uint_fixed_16_16_t plane_blocks_per_line)
5063 {
5064         u32 wm_intermediate_val;
5065         uint_fixed_16_16_t ret;
5066
5067         if (latency == 0)
5068                 return FP_16_16_MAX;
5069
5070         wm_intermediate_val = latency * pixel_rate;
5071         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5072                                            pipe_htotal * 1000);
5073         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5074         return ret;
5075 }
5076
5077 static uint_fixed_16_16_t
5078 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5079 {
5080         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5081         u32 pixel_rate;
5082         u32 crtc_htotal;
5083         uint_fixed_16_16_t linetime_us;
5084
5085         if (!crtc_state->hw.active)
5086                 return u32_to_fixed16(0);
5087
5088         pixel_rate = crtc_state->pixel_rate;
5089
5090         if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5091                 return u32_to_fixed16(0);
5092
5093         crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5094         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5095
5096         return linetime_us;
5097 }
5098
5099 static u32
5100 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5101                               const struct intel_plane_state *plane_state)
5102 {
5103         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5104         u64 adjusted_pixel_rate;
5105         uint_fixed_16_16_t downscale_amount;
5106
5107         /* Shouldn't reach here on disabled planes... */
5108         if (drm_WARN_ON(&dev_priv->drm,
5109                         !intel_wm_plane_visible(crtc_state, plane_state)))
5110                 return 0;
5111
5112         /*
5113          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5114          * with additional adjustments for plane-specific scaling.
5115          */
5116         adjusted_pixel_rate = crtc_state->pixel_rate;
5117         downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5118
5119         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5120                                             downscale_amount);
5121 }
5122
5123 static int
5124 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5125                       int width, const struct drm_format_info *format,
5126                       u64 modifier, unsigned int rotation,
5127                       u32 plane_pixel_rate, struct skl_wm_params *wp,
5128                       int color_plane)
5129 {
5130         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5131         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5132         u32 interm_pbpl;
5133
5134         /* only planar format has two planes */
5135         if (color_plane == 1 &&
5136             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5137                 drm_dbg_kms(&dev_priv->drm,
5138                             "Non planar format have single plane\n");
5139                 return -EINVAL;
5140         }
5141
5142         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5143                       modifier == I915_FORMAT_MOD_Yf_TILED ||
5144                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5145                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5146         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5147         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5148                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5149         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5150
5151         wp->width = width;
5152         if (color_plane == 1 && wp->is_planar)
5153                 wp->width /= 2;
5154
5155         wp->cpp = format->cpp[color_plane];
5156         wp->plane_pixel_rate = plane_pixel_rate;
5157
5158         if (INTEL_GEN(dev_priv) >= 11 &&
5159             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5160                 wp->dbuf_block_size = 256;
5161         else
5162                 wp->dbuf_block_size = 512;
5163
5164         if (drm_rotation_90_or_270(rotation)) {
5165                 switch (wp->cpp) {
5166                 case 1:
5167                         wp->y_min_scanlines = 16;
5168                         break;
5169                 case 2:
5170                         wp->y_min_scanlines = 8;
5171                         break;
5172                 case 4:
5173                         wp->y_min_scanlines = 4;
5174                         break;
5175                 default:
5176                         MISSING_CASE(wp->cpp);
5177                         return -EINVAL;
5178                 }
5179         } else {
5180                 wp->y_min_scanlines = 4;
5181         }
5182
5183         if (skl_needs_memory_bw_wa(dev_priv))
5184                 wp->y_min_scanlines *= 2;
5185
5186         wp->plane_bytes_per_line = wp->width * wp->cpp;
5187         if (wp->y_tiled) {
5188                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5189                                            wp->y_min_scanlines,
5190                                            wp->dbuf_block_size);
5191
5192                 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5193                         interm_pbpl++;
5194
5195                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5196                                                         wp->y_min_scanlines);
5197         } else {
5198                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5199                                            wp->dbuf_block_size);
5200
5201                 if (!wp->x_tiled ||
5202                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5203                         interm_pbpl++;
5204
5205                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5206         }
5207
5208         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5209                                              wp->plane_blocks_per_line);
5210
5211         wp->linetime_us = fixed16_to_u32_round_up(
5212                                         intel_get_linetime_us(crtc_state));
5213
5214         return 0;
5215 }
5216
5217 static int
5218 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5219                             const struct intel_plane_state *plane_state,
5220                             struct skl_wm_params *wp, int color_plane)
5221 {
5222         const struct drm_framebuffer *fb = plane_state->hw.fb;
5223         int width;
5224
5225         /*
5226          * Src coordinates are already rotated by 270 degrees for
5227          * the 90/270 degree plane rotation cases (to match the
5228          * GTT mapping), hence no need to account for rotation here.
5229          */
5230         width = drm_rect_width(&plane_state->uapi.src) >> 16;
5231
5232         return skl_compute_wm_params(crtc_state, width,
5233                                      fb->format, fb->modifier,
5234                                      plane_state->hw.rotation,
5235                                      skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5236                                      wp, color_plane);
5237 }
5238
5239 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5240 {
5241         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5242                 return true;
5243
5244         /* The number of lines are ignored for the level 0 watermark. */
5245         return level > 0;
5246 }
5247
5248 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5249                                  int level,
5250                                  unsigned int latency,
5251                                  const struct skl_wm_params *wp,
5252                                  const struct skl_wm_level *result_prev,
5253                                  struct skl_wm_level *result /* out */)
5254 {
5255         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5256         uint_fixed_16_16_t method1, method2;
5257         uint_fixed_16_16_t selected_result;
5258         u32 res_blocks, res_lines, min_ddb_alloc = 0;
5259
5260         if (latency == 0) {
5261                 /* reject it */
5262                 result->min_ddb_alloc = U16_MAX;
5263                 return;
5264         }
5265
5266         /*
5267          * WaIncreaseLatencyIPCEnabled: kbl,cfl
5268          * Display WA #1141: kbl,cfl
5269          */
5270         if ((IS_KABYLAKE(dev_priv) ||
5271              IS_COFFEELAKE(dev_priv) ||
5272              IS_COMETLAKE(dev_priv)) &&
5273             dev_priv->ipc_enabled)
5274                 latency += 4;
5275
5276         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5277                 latency += 15;
5278
5279         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5280                                  wp->cpp, latency, wp->dbuf_block_size);
5281         method2 = skl_wm_method2(wp->plane_pixel_rate,
5282                                  crtc_state->hw.adjusted_mode.crtc_htotal,
5283                                  latency,
5284                                  wp->plane_blocks_per_line);
5285
5286         if (wp->y_tiled) {
5287                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5288         } else {
5289                 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5290                      wp->dbuf_block_size < 1) &&
5291                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5292                         selected_result = method2;
5293                 } else if (latency >= wp->linetime_us) {
5294                         if (IS_GEN(dev_priv, 9) &&
5295                             !IS_GEMINILAKE(dev_priv))
5296                                 selected_result = min_fixed16(method1, method2);
5297                         else
5298                                 selected_result = method2;
5299                 } else {
5300                         selected_result = method1;
5301                 }
5302         }
5303
5304         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5305         res_lines = div_round_up_fixed16(selected_result,
5306                                          wp->plane_blocks_per_line);
5307
5308         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5309                 /* Display WA #1125: skl,bxt,kbl */
5310                 if (level == 0 && wp->rc_surface)
5311                         res_blocks +=
5312                                 fixed16_to_u32_round_up(wp->y_tile_minimum);
5313
5314                 /* Display WA #1126: skl,bxt,kbl */
5315                 if (level >= 1 && level <= 7) {
5316                         if (wp->y_tiled) {
5317                                 res_blocks +=
5318                                     fixed16_to_u32_round_up(wp->y_tile_minimum);
5319                                 res_lines += wp->y_min_scanlines;
5320                         } else {
5321                                 res_blocks++;
5322                         }
5323
5324                         /*
5325                          * Make sure result blocks for higher latency levels are
5326                          * atleast as high as level below the current level.
5327                          * Assumption in DDB algorithm optimization for special
5328                          * cases. Also covers Display WA #1125 for RC.
5329                          */
5330                         if (result_prev->plane_res_b > res_blocks)
5331                                 res_blocks = result_prev->plane_res_b;
5332                 }
5333         }
5334
5335         if (INTEL_GEN(dev_priv) >= 11) {
5336                 if (wp->y_tiled) {
5337                         int extra_lines;
5338
5339                         if (res_lines % wp->y_min_scanlines == 0)
5340                                 extra_lines = wp->y_min_scanlines;
5341                         else
5342                                 extra_lines = wp->y_min_scanlines * 2 -
5343                                         res_lines % wp->y_min_scanlines;
5344
5345                         min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5346                                                                  wp->plane_blocks_per_line);
5347                 } else {
5348                         min_ddb_alloc = res_blocks +
5349                                 DIV_ROUND_UP(res_blocks, 10);
5350                 }
5351         }
5352
5353         if (!skl_wm_has_lines(dev_priv, level))
5354                 res_lines = 0;
5355
5356         if (res_lines > 31) {
5357                 /* reject it */
5358                 result->min_ddb_alloc = U16_MAX;
5359                 return;
5360         }
5361
5362         /*
5363          * If res_lines is valid, assume we can use this watermark level
5364          * for now.  We'll come back and disable it after we calculate the
5365          * DDB allocation if it turns out we don't actually have enough
5366          * blocks to satisfy it.
5367          */
5368         result->plane_res_b = res_blocks;
5369         result->plane_res_l = res_lines;
5370         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5371         result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5372         result->plane_en = true;
5373 }
5374
5375 static void
5376 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5377                       const struct skl_wm_params *wm_params,
5378                       struct skl_wm_level *levels)
5379 {
5380         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5381         int level, max_level = ilk_wm_max_level(dev_priv);
5382         struct skl_wm_level *result_prev = &levels[0];
5383
5384         for (level = 0; level <= max_level; level++) {
5385                 struct skl_wm_level *result = &levels[level];
5386                 unsigned int latency = dev_priv->wm.skl_latency[level];
5387
5388                 skl_compute_plane_wm(crtc_state, level, latency,
5389                                      wm_params, result_prev, result);
5390
5391                 result_prev = result;
5392         }
5393 }
5394
5395 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5396                                 const struct skl_wm_params *wm_params,
5397                                 struct skl_plane_wm *plane_wm)
5398 {
5399         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5400         struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5401         struct skl_wm_level *levels = plane_wm->wm;
5402         unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5403
5404         skl_compute_plane_wm(crtc_state, 0, latency,
5405                              wm_params, &levels[0],
5406                              sagv_wm);
5407 }
5408
5409 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5410                                       const struct skl_wm_params *wp,
5411                                       struct skl_plane_wm *wm)
5412 {
5413         struct drm_device *dev = crtc_state->uapi.crtc->dev;
5414         const struct drm_i915_private *dev_priv = to_i915(dev);
5415         u16 trans_min, trans_amount, trans_y_tile_min;
5416         u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5417
5418         /* Transition WM don't make any sense if ipc is disabled */
5419         if (!dev_priv->ipc_enabled)
5420                 return;
5421
5422         /*
5423          * WaDisableTWM:skl,kbl,cfl,bxt
5424          * Transition WM are not recommended by HW team for GEN9
5425          */
5426         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5427                 return;
5428
5429         if (INTEL_GEN(dev_priv) >= 11)
5430                 trans_min = 4;
5431         else
5432                 trans_min = 14;
5433
5434         /* Display WA #1140: glk,cnl */
5435         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5436                 trans_amount = 0;
5437         else
5438                 trans_amount = 10; /* This is configurable amount */
5439
5440         trans_offset_b = trans_min + trans_amount;
5441
5442         /*
5443          * The spec asks for Selected Result Blocks for wm0 (the real value),
5444          * not Result Blocks (the integer value). Pay attention to the capital
5445          * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5446          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5447          * and since we later will have to get the ceiling of the sum in the
5448          * transition watermarks calculation, we can just pretend Selected
5449          * Result Blocks is Result Blocks minus 1 and it should work for the
5450          * current platforms.
5451          */
5452         wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5453
5454         if (wp->y_tiled) {
5455                 trans_y_tile_min =
5456                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5457                 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5458                                 trans_offset_b;
5459         } else {
5460                 res_blocks = wm0_sel_res_b + trans_offset_b;
5461         }
5462
5463         /*
5464          * Just assume we can enable the transition watermark.  After
5465          * computing the DDB we'll come back and disable it if that
5466          * assumption turns out to be false.
5467          */
5468         wm->trans_wm.plane_res_b = res_blocks + 1;
5469         wm->trans_wm.plane_en = true;
5470 }
5471
5472 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5473                                      const struct intel_plane_state *plane_state,
5474                                      enum plane_id plane_id, int color_plane)
5475 {
5476         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5477         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5478         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5479         struct skl_wm_params wm_params;
5480         int ret;
5481
5482         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5483                                           &wm_params, color_plane);
5484         if (ret)
5485                 return ret;
5486
5487         skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5488
5489         if (INTEL_GEN(dev_priv) >= 12)
5490                 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5491
5492         skl_compute_transition_wm(crtc_state, &wm_params, wm);
5493
5494         return 0;
5495 }
5496
5497 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5498                                  const struct intel_plane_state *plane_state,
5499                                  enum plane_id plane_id)
5500 {
5501         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5502         struct skl_wm_params wm_params;
5503         int ret;
5504
5505         wm->is_planar = true;
5506
5507         /* uv plane watermarks must also be validated for NV12/Planar */
5508         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5509                                           &wm_params, 1);
5510         if (ret)
5511                 return ret;
5512
5513         skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5514
5515         return 0;
5516 }
5517
5518 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5519                               const struct intel_plane_state *plane_state)
5520 {
5521         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5522         const struct drm_framebuffer *fb = plane_state->hw.fb;
5523         enum plane_id plane_id = plane->id;
5524         int ret;
5525
5526         if (!intel_wm_plane_visible(crtc_state, plane_state))
5527                 return 0;
5528
5529         ret = skl_build_plane_wm_single(crtc_state, plane_state,
5530                                         plane_id, 0);
5531         if (ret)
5532                 return ret;
5533
5534         if (fb->format->is_yuv && fb->format->num_planes > 1) {
5535                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5536                                             plane_id);
5537                 if (ret)
5538                         return ret;
5539         }
5540
5541         return 0;
5542 }
5543
5544 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5545                               const struct intel_plane_state *plane_state)
5546 {
5547         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5548         enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5549         int ret;
5550
5551         /* Watermarks calculated in master */
5552         if (plane_state->planar_slave)
5553                 return 0;
5554
5555         if (plane_state->planar_linked_plane) {
5556                 const struct drm_framebuffer *fb = plane_state->hw.fb;
5557                 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5558
5559                 drm_WARN_ON(&dev_priv->drm,
5560                             !intel_wm_plane_visible(crtc_state, plane_state));
5561                 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5562                             fb->format->num_planes == 1);
5563
5564                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5565                                                 y_plane_id, 0);
5566                 if (ret)
5567                         return ret;
5568
5569                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5570                                                 plane_id, 1);
5571                 if (ret)
5572                         return ret;
5573         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5574                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5575                                                 plane_id, 0);
5576                 if (ret)
5577                         return ret;
5578         }
5579
5580         return 0;
5581 }
5582
5583 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5584 {
5585         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5586         struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5587         struct intel_plane *plane;
5588         const struct intel_plane_state *plane_state;
5589         int ret;
5590
5591         /*
5592          * We'll only calculate watermarks for planes that are actually
5593          * enabled, so make sure all other planes are set as disabled.
5594          */
5595         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5596
5597         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5598                                                      crtc_state) {
5599
5600                 if (INTEL_GEN(dev_priv) >= 11)
5601                         ret = icl_build_plane_wm(crtc_state, plane_state);
5602                 else
5603                         ret = skl_build_plane_wm(crtc_state, plane_state);
5604                 if (ret)
5605                         return ret;
5606         }
5607
5608         return 0;
5609 }
5610
5611 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5612                                 i915_reg_t reg,
5613                                 const struct skl_ddb_entry *entry)
5614 {
5615         if (entry->end)
5616                 intel_de_write_fw(dev_priv, reg,
5617                                   (entry->end - 1) << 16 | entry->start);
5618         else
5619                 intel_de_write_fw(dev_priv, reg, 0);
5620 }
5621
5622 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5623                                i915_reg_t reg,
5624                                const struct skl_wm_level *level)
5625 {
5626         u32 val = 0;
5627
5628         if (level->plane_en)
5629                 val |= PLANE_WM_EN;
5630         if (level->ignore_lines)
5631                 val |= PLANE_WM_IGNORE_LINES;
5632         val |= level->plane_res_b;
5633         val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5634
5635         intel_de_write_fw(dev_priv, reg, val);
5636 }
5637
5638 void skl_write_plane_wm(struct intel_plane *plane,
5639                         const struct intel_crtc_state *crtc_state)
5640 {
5641         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5642         int level, max_level = ilk_wm_max_level(dev_priv);
5643         enum plane_id plane_id = plane->id;
5644         enum pipe pipe = plane->pipe;
5645         const struct skl_plane_wm *wm =
5646                 &crtc_state->wm.skl.optimal.planes[plane_id];
5647         const struct skl_ddb_entry *ddb_y =
5648                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5649         const struct skl_ddb_entry *ddb_uv =
5650                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5651
5652         for (level = 0; level <= max_level; level++) {
5653                 const struct skl_wm_level *wm_level;
5654
5655                 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5656
5657                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5658                                    wm_level);
5659         }
5660         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5661                            &wm->trans_wm);
5662
5663         if (INTEL_GEN(dev_priv) >= 11) {
5664                 skl_ddb_entry_write(dev_priv,
5665                                     PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5666                 return;
5667         }
5668
5669         if (wm->is_planar)
5670                 swap(ddb_y, ddb_uv);
5671
5672         skl_ddb_entry_write(dev_priv,
5673                             PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5674         skl_ddb_entry_write(dev_priv,
5675                             PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5676 }
5677
5678 void skl_write_cursor_wm(struct intel_plane *plane,
5679                          const struct intel_crtc_state *crtc_state)
5680 {
5681         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5682         int level, max_level = ilk_wm_max_level(dev_priv);
5683         enum plane_id plane_id = plane->id;
5684         enum pipe pipe = plane->pipe;
5685         const struct skl_plane_wm *wm =
5686                 &crtc_state->wm.skl.optimal.planes[plane_id];
5687         const struct skl_ddb_entry *ddb =
5688                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5689
5690         for (level = 0; level <= max_level; level++) {
5691                 const struct skl_wm_level *wm_level;
5692
5693                 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5694
5695                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5696                                    wm_level);
5697         }
5698         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5699
5700         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5701 }
5702
5703 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5704                          const struct skl_wm_level *l2)
5705 {
5706         return l1->plane_en == l2->plane_en &&
5707                 l1->ignore_lines == l2->ignore_lines &&
5708                 l1->plane_res_l == l2->plane_res_l &&
5709                 l1->plane_res_b == l2->plane_res_b;
5710 }
5711
5712 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5713                                 const struct skl_plane_wm *wm1,
5714                                 const struct skl_plane_wm *wm2)
5715 {
5716         int level, max_level = ilk_wm_max_level(dev_priv);
5717
5718         for (level = 0; level <= max_level; level++) {
5719                 /*
5720                  * We don't check uv_wm as the hardware doesn't actually
5721                  * use it. It only gets used for calculating the required
5722                  * ddb allocation.
5723                  */
5724                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5725                         return false;
5726         }
5727
5728         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5729 }
5730
5731 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5732                                     const struct skl_ddb_entry *b)
5733 {
5734         return a->start < b->end && b->start < a->end;
5735 }
5736
5737 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5738                                  const struct skl_ddb_entry *entries,
5739                                  int num_entries, int ignore_idx)
5740 {
5741         int i;
5742
5743         for (i = 0; i < num_entries; i++) {
5744                 if (i != ignore_idx &&
5745                     skl_ddb_entries_overlap(ddb, &entries[i]))
5746                         return true;
5747         }
5748
5749         return false;
5750 }
5751
5752 static int
5753 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5754                             struct intel_crtc_state *new_crtc_state)
5755 {
5756         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5757         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5758         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5759         struct intel_plane *plane;
5760
5761         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5762                 struct intel_plane_state *plane_state;
5763                 enum plane_id plane_id = plane->id;
5764
5765                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5766                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5767                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5768                                         &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5769                         continue;
5770
5771                 plane_state = intel_atomic_get_plane_state(state, plane);
5772                 if (IS_ERR(plane_state))
5773                         return PTR_ERR(plane_state);
5774
5775                 new_crtc_state->update_planes |= BIT(plane_id);
5776         }
5777
5778         return 0;
5779 }
5780
5781 static int
5782 skl_compute_ddb(struct intel_atomic_state *state)
5783 {
5784         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5785         const struct intel_dbuf_state *old_dbuf_state;
5786         const struct intel_dbuf_state *new_dbuf_state;
5787         const struct intel_crtc_state *old_crtc_state;
5788         struct intel_crtc_state *new_crtc_state;
5789         struct intel_crtc *crtc;
5790         int ret, i;
5791
5792         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5793                                             new_crtc_state, i) {
5794                 ret = skl_allocate_pipe_ddb(new_crtc_state);
5795                 if (ret)
5796                         return ret;
5797
5798                 ret = skl_ddb_add_affected_planes(old_crtc_state,
5799                                                   new_crtc_state);
5800                 if (ret)
5801                         return ret;
5802         }
5803
5804         old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5805         new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
5806
5807         if (new_dbuf_state &&
5808             new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices)
5809                 drm_dbg_kms(&dev_priv->drm,
5810                             "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5811                             old_dbuf_state->enabled_slices,
5812                             new_dbuf_state->enabled_slices,
5813                             INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5814
5815         return 0;
5816 }
5817
5818 static char enast(bool enable)
5819 {
5820         return enable ? '*' : ' ';
5821 }
5822
5823 static void
5824 skl_print_wm_changes(struct intel_atomic_state *state)
5825 {
5826         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5827         const struct intel_crtc_state *old_crtc_state;
5828         const struct intel_crtc_state *new_crtc_state;
5829         struct intel_plane *plane;
5830         struct intel_crtc *crtc;
5831         int i;
5832
5833         if (!drm_debug_enabled(DRM_UT_KMS))
5834                 return;
5835
5836         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5837                                             new_crtc_state, i) {
5838                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5839
5840                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5841                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5842
5843                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5844                         enum plane_id plane_id = plane->id;
5845                         const struct skl_ddb_entry *old, *new;
5846
5847                         old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5848                         new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5849
5850                         if (skl_ddb_entry_equal(old, new))
5851                                 continue;
5852
5853                         drm_dbg_kms(&dev_priv->drm,
5854                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5855                                     plane->base.base.id, plane->base.name,
5856                                     old->start, old->end, new->start, new->end,
5857                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5858                 }
5859
5860                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5861                         enum plane_id plane_id = plane->id;
5862                         const struct skl_plane_wm *old_wm, *new_wm;
5863
5864                         old_wm = &old_pipe_wm->planes[plane_id];
5865                         new_wm = &new_pipe_wm->planes[plane_id];
5866
5867                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5868                                 continue;
5869
5870                         drm_dbg_kms(&dev_priv->drm,
5871                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5872                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5873                                     plane->base.base.id, plane->base.name,
5874                                     enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5875                                     enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5876                                     enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5877                                     enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5878                                     enast(old_wm->trans_wm.plane_en),
5879                                     enast(old_wm->sagv_wm0.plane_en),
5880                                     enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5881                                     enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5882                                     enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5883                                     enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5884                                     enast(new_wm->trans_wm.plane_en),
5885                                     enast(new_wm->sagv_wm0.plane_en));
5886
5887                         drm_dbg_kms(&dev_priv->drm,
5888                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5889                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5890                                     plane->base.base.id, plane->base.name,
5891                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5892                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5893                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5894                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5895                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5896                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5897                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5898                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5899                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5900                                     enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5901
5902                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5903                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5904                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5905                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5906                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5907                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5908                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5909                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5910                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5911                                     enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5912
5913                         drm_dbg_kms(&dev_priv->drm,
5914                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5915                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5916                                     plane->base.base.id, plane->base.name,
5917                                     old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5918                                     old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5919                                     old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5920                                     old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5921                                     old_wm->trans_wm.plane_res_b,
5922                                     old_wm->sagv_wm0.plane_res_b,
5923                                     new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5924                                     new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5925                                     new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5926                                     new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5927                                     new_wm->trans_wm.plane_res_b,
5928                                     new_wm->sagv_wm0.plane_res_b);
5929
5930                         drm_dbg_kms(&dev_priv->drm,
5931                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5932                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5933                                     plane->base.base.id, plane->base.name,
5934                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5935                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5936                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5937                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5938                                     old_wm->trans_wm.min_ddb_alloc,
5939                                     old_wm->sagv_wm0.min_ddb_alloc,
5940                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5941                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5942                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5943                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5944                                     new_wm->trans_wm.min_ddb_alloc,
5945                                     new_wm->sagv_wm0.min_ddb_alloc);
5946                 }
5947         }
5948 }
5949
5950 static int intel_add_affected_pipes(struct intel_atomic_state *state,
5951                                     u8 pipe_mask)
5952 {
5953         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5954         struct intel_crtc *crtc;
5955
5956         for_each_intel_crtc(&dev_priv->drm, crtc) {
5957                 struct intel_crtc_state *crtc_state;
5958
5959                 if ((pipe_mask & BIT(crtc->pipe)) == 0)
5960                         continue;
5961
5962                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5963                 if (IS_ERR(crtc_state))
5964                         return PTR_ERR(crtc_state);
5965         }
5966
5967         return 0;
5968 }
5969
5970 static int
5971 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5972 {
5973         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5974         struct intel_crtc_state *crtc_state;
5975         struct intel_crtc *crtc;
5976         int i, ret;
5977
5978         if (dev_priv->wm.distrust_bios_wm) {
5979                 /*
5980                  * skl_ddb_get_pipe_allocation_limits() currently requires
5981                  * all active pipes to be included in the state so that
5982                  * it can redistribute the dbuf among them, and it really
5983                  * wants to recompute things when distrust_bios_wm is set
5984                  * so we add all the pipes to the state.
5985                  */
5986                 ret = intel_add_affected_pipes(state, ~0);
5987                 if (ret)
5988                         return ret;
5989         }
5990
5991         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
5992                 struct intel_dbuf_state *new_dbuf_state;
5993                 const struct intel_dbuf_state *old_dbuf_state;
5994
5995                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5996                 if (IS_ERR(new_dbuf_state))
5997                         return PTR_ERR(new_dbuf_state);
5998
5999                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6000
6001                 new_dbuf_state->active_pipes =
6002                         intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6003
6004                 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes)
6005                         break;
6006
6007                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6008                 if (ret)
6009                         return ret;
6010
6011                 /*
6012                  * skl_ddb_get_pipe_allocation_limits() currently requires
6013                  * all active pipes to be included in the state so that
6014                  * it can redistribute the dbuf among them.
6015                  */
6016                 ret = intel_add_affected_pipes(state,
6017                                                new_dbuf_state->active_pipes);
6018                 if (ret)
6019                         return ret;
6020
6021                 break;
6022         }
6023
6024         return 0;
6025 }
6026
6027 /*
6028  * To make sure the cursor watermark registers are always consistent
6029  * with our computed state the following scenario needs special
6030  * treatment:
6031  *
6032  * 1. enable cursor
6033  * 2. move cursor entirely offscreen
6034  * 3. disable cursor
6035  *
6036  * Step 2. does call .disable_plane() but does not zero the watermarks
6037  * (since we consider an offscreen cursor still active for the purposes
6038  * of watermarks). Step 3. would not normally call .disable_plane()
6039  * because the actual plane visibility isn't changing, and we don't
6040  * deallocate the cursor ddb until the pipe gets disabled. So we must
6041  * force step 3. to call .disable_plane() to update the watermark
6042  * registers properly.
6043  *
6044  * Other planes do not suffer from this issues as their watermarks are
6045  * calculated based on the actual plane visibility. The only time this
6046  * can trigger for the other planes is during the initial readout as the
6047  * default value of the watermarks registers is not zero.
6048  */
6049 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6050                                       struct intel_crtc *crtc)
6051 {
6052         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6053         const struct intel_crtc_state *old_crtc_state =
6054                 intel_atomic_get_old_crtc_state(state, crtc);
6055         struct intel_crtc_state *new_crtc_state =
6056                 intel_atomic_get_new_crtc_state(state, crtc);
6057         struct intel_plane *plane;
6058
6059         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6060                 struct intel_plane_state *plane_state;
6061                 enum plane_id plane_id = plane->id;
6062
6063                 /*
6064                  * Force a full wm update for every plane on modeset.
6065                  * Required because the reset value of the wm registers
6066                  * is non-zero, whereas we want all disabled planes to
6067                  * have zero watermarks. So if we turn off the relevant
6068                  * power well the hardware state will go out of sync
6069                  * with the software state.
6070                  */
6071                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6072                     skl_plane_wm_equals(dev_priv,
6073                                         &old_crtc_state->wm.skl.optimal.planes[plane_id],
6074                                         &new_crtc_state->wm.skl.optimal.planes[plane_id]))
6075                         continue;
6076
6077                 plane_state = intel_atomic_get_plane_state(state, plane);
6078                 if (IS_ERR(plane_state))
6079                         return PTR_ERR(plane_state);
6080
6081                 new_crtc_state->update_planes |= BIT(plane_id);
6082         }
6083
6084         return 0;
6085 }
6086
6087 static int
6088 skl_compute_wm(struct intel_atomic_state *state)
6089 {
6090         struct intel_crtc *crtc;
6091         struct intel_crtc_state *new_crtc_state;
6092         struct intel_crtc_state *old_crtc_state;
6093         int ret, i;
6094
6095         ret = skl_ddb_add_affected_pipes(state);
6096         if (ret)
6097                 return ret;
6098
6099         /*
6100          * Calculate WM's for all pipes that are part of this transaction.
6101          * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6102          * weren't otherwise being modified if pipe allocations had to change.
6103          */
6104         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6105                                             new_crtc_state, i) {
6106                 ret = skl_build_pipe_wm(new_crtc_state);
6107                 if (ret)
6108                         return ret;
6109         }
6110
6111         ret = skl_compute_ddb(state);
6112         if (ret)
6113                 return ret;
6114
6115         ret = intel_compute_sagv_mask(state);
6116         if (ret)
6117                 return ret;
6118
6119         /*
6120          * skl_compute_ddb() will have adjusted the final watermarks
6121          * based on how much ddb is available. Now we can actually
6122          * check if the final watermarks changed.
6123          */
6124         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6125                                             new_crtc_state, i) {
6126                 ret = skl_wm_add_affected_planes(state, crtc);
6127                 if (ret)
6128                         return ret;
6129         }
6130
6131         skl_print_wm_changes(state);
6132
6133         return 0;
6134 }
6135
6136 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6137                                   struct intel_wm_config *config)
6138 {
6139         struct intel_crtc *crtc;
6140
6141         /* Compute the currently _active_ config */
6142         for_each_intel_crtc(&dev_priv->drm, crtc) {
6143                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6144
6145                 if (!wm->pipe_enabled)
6146                         continue;
6147
6148                 config->sprites_enabled |= wm->sprites_enabled;
6149                 config->sprites_scaled |= wm->sprites_scaled;
6150                 config->num_pipes_active++;
6151         }
6152 }
6153
6154 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6155 {
6156         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6157         struct ilk_wm_maximums max;
6158         struct intel_wm_config config = {};
6159         struct ilk_wm_values results = {};
6160         enum intel_ddb_partitioning partitioning;
6161
6162         ilk_compute_wm_config(dev_priv, &config);
6163
6164         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6165         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6166
6167         /* 5/6 split only in single pipe config on IVB+ */
6168         if (INTEL_GEN(dev_priv) >= 7 &&
6169             config.num_pipes_active == 1 && config.sprites_enabled) {
6170                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6171                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6172
6173                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6174         } else {
6175                 best_lp_wm = &lp_wm_1_2;
6176         }
6177
6178         partitioning = (best_lp_wm == &lp_wm_1_2) ?
6179                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6180
6181         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6182
6183         ilk_write_wm_values(dev_priv, &results);
6184 }
6185
6186 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6187                                    struct intel_crtc *crtc)
6188 {
6189         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6190         const struct intel_crtc_state *crtc_state =
6191                 intel_atomic_get_new_crtc_state(state, crtc);
6192
6193         mutex_lock(&dev_priv->wm.wm_mutex);
6194         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6195         ilk_program_watermarks(dev_priv);
6196         mutex_unlock(&dev_priv->wm.wm_mutex);
6197 }
6198
6199 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6200                                     struct intel_crtc *crtc)
6201 {
6202         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6203         const struct intel_crtc_state *crtc_state =
6204                 intel_atomic_get_new_crtc_state(state, crtc);
6205
6206         if (!crtc_state->wm.need_postvbl_update)
6207                 return;
6208
6209         mutex_lock(&dev_priv->wm.wm_mutex);
6210         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6211         ilk_program_watermarks(dev_priv);
6212         mutex_unlock(&dev_priv->wm.wm_mutex);
6213 }
6214
6215 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6216 {
6217         level->plane_en = val & PLANE_WM_EN;
6218         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6219         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6220         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6221                 PLANE_WM_LINES_MASK;
6222 }
6223
6224 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6225                               struct skl_pipe_wm *out)
6226 {
6227         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6228         enum pipe pipe = crtc->pipe;
6229         int level, max_level;
6230         enum plane_id plane_id;
6231         u32 val;
6232
6233         max_level = ilk_wm_max_level(dev_priv);
6234
6235         for_each_plane_id_on_crtc(crtc, plane_id) {
6236                 struct skl_plane_wm *wm = &out->planes[plane_id];
6237
6238                 for (level = 0; level <= max_level; level++) {
6239                         if (plane_id != PLANE_CURSOR)
6240                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6241                         else
6242                                 val = I915_READ(CUR_WM(pipe, level));
6243
6244                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
6245                 }
6246
6247                 if (INTEL_GEN(dev_priv) >= 12)
6248                         wm->sagv_wm0 = wm->wm[0];
6249
6250                 if (plane_id != PLANE_CURSOR)
6251                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6252                 else
6253                         val = I915_READ(CUR_WM_TRANS(pipe));
6254
6255                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6256         }
6257
6258         if (!crtc->active)
6259                 return;
6260 }
6261
6262 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6263 {
6264         struct intel_crtc *crtc;
6265         struct intel_crtc_state *crtc_state;
6266
6267         for_each_intel_crtc(&dev_priv->drm, crtc) {
6268                 crtc_state = to_intel_crtc_state(crtc->base.state);
6269
6270                 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6271         }
6272
6273         if (dev_priv->active_pipes) {
6274                 /* Fully recompute DDB on first atomic commit */
6275                 dev_priv->wm.distrust_bios_wm = true;
6276         }
6277 }
6278
6279 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6280 {
6281         struct drm_device *dev = crtc->base.dev;
6282         struct drm_i915_private *dev_priv = to_i915(dev);
6283         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6284         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6285         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6286         enum pipe pipe = crtc->pipe;
6287         static const i915_reg_t wm0_pipe_reg[] = {
6288                 [PIPE_A] = WM0_PIPEA_ILK,
6289                 [PIPE_B] = WM0_PIPEB_ILK,
6290                 [PIPE_C] = WM0_PIPEC_IVB,
6291         };
6292
6293         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6294
6295         memset(active, 0, sizeof(*active));
6296
6297         active->pipe_enabled = crtc->active;
6298
6299         if (active->pipe_enabled) {
6300                 u32 tmp = hw->wm_pipe[pipe];
6301
6302                 /*
6303                  * For active pipes LP0 watermark is marked as
6304                  * enabled, and LP1+ watermaks as disabled since
6305                  * we can't really reverse compute them in case
6306                  * multiple pipes are active.
6307                  */
6308                 active->wm[0].enable = true;
6309                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6310                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6311                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6312         } else {
6313                 int level, max_level = ilk_wm_max_level(dev_priv);
6314
6315                 /*
6316                  * For inactive pipes, all watermark levels
6317                  * should be marked as enabled but zeroed,
6318                  * which is what we'd compute them to.
6319                  */
6320                 for (level = 0; level <= max_level; level++)
6321                         active->wm[level].enable = true;
6322         }
6323
6324         crtc->wm.active.ilk = *active;
6325 }
6326
6327 #define _FW_WM(value, plane) \
6328         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6329 #define _FW_WM_VLV(value, plane) \
6330         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6331
6332 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6333                                struct g4x_wm_values *wm)
6334 {
6335         u32 tmp;
6336
6337         tmp = I915_READ(DSPFW1);
6338         wm->sr.plane = _FW_WM(tmp, SR);
6339         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6340         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6341         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6342
6343         tmp = I915_READ(DSPFW2);
6344         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6345         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6346         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6347         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6348         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6349         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6350
6351         tmp = I915_READ(DSPFW3);
6352         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6353         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6354         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6355         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6356 }
6357
6358 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6359                                struct vlv_wm_values *wm)
6360 {
6361         enum pipe pipe;
6362         u32 tmp;
6363
6364         for_each_pipe(dev_priv, pipe) {
6365                 tmp = I915_READ(VLV_DDL(pipe));
6366
6367                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6368                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6369                 wm->ddl[pipe].plane[PLANE_CURSOR] =
6370                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6371                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6372                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6373                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6374                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6375         }
6376
6377         tmp = I915_READ(DSPFW1);
6378         wm->sr.plane = _FW_WM(tmp, SR);
6379         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6380         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6381         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6382
6383         tmp = I915_READ(DSPFW2);
6384         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6385         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6386         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6387
6388         tmp = I915_READ(DSPFW3);
6389         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6390
6391         if (IS_CHERRYVIEW(dev_priv)) {
6392                 tmp = I915_READ(DSPFW7_CHV);
6393                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6394                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6395
6396                 tmp = I915_READ(DSPFW8_CHV);
6397                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6398                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6399
6400                 tmp = I915_READ(DSPFW9_CHV);
6401                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6402                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6403
6404                 tmp = I915_READ(DSPHOWM);
6405                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6406                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6407                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6408                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6409                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6410                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6411                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6412                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6413                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6414                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6415         } else {
6416                 tmp = I915_READ(DSPFW7);
6417                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6418                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6419
6420                 tmp = I915_READ(DSPHOWM);
6421                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6422                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6423                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6424                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6425                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6426                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6427                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6428         }
6429 }
6430
6431 #undef _FW_WM
6432 #undef _FW_WM_VLV
6433
6434 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6435 {
6436         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6437         struct intel_crtc *crtc;
6438
6439         g4x_read_wm_values(dev_priv, wm);
6440
6441         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6442
6443         for_each_intel_crtc(&dev_priv->drm, crtc) {
6444                 struct intel_crtc_state *crtc_state =
6445                         to_intel_crtc_state(crtc->base.state);
6446                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6447                 struct g4x_pipe_wm *raw;
6448                 enum pipe pipe = crtc->pipe;
6449                 enum plane_id plane_id;
6450                 int level, max_level;
6451
6452                 active->cxsr = wm->cxsr;
6453                 active->hpll_en = wm->hpll_en;
6454                 active->fbc_en = wm->fbc_en;
6455
6456                 active->sr = wm->sr;
6457                 active->hpll = wm->hpll;
6458
6459                 for_each_plane_id_on_crtc(crtc, plane_id) {
6460                         active->wm.plane[plane_id] =
6461                                 wm->pipe[pipe].plane[plane_id];
6462                 }
6463
6464                 if (wm->cxsr && wm->hpll_en)
6465                         max_level = G4X_WM_LEVEL_HPLL;
6466                 else if (wm->cxsr)
6467                         max_level = G4X_WM_LEVEL_SR;
6468                 else
6469                         max_level = G4X_WM_LEVEL_NORMAL;
6470
6471                 level = G4X_WM_LEVEL_NORMAL;
6472                 raw = &crtc_state->wm.g4x.raw[level];
6473                 for_each_plane_id_on_crtc(crtc, plane_id)
6474                         raw->plane[plane_id] = active->wm.plane[plane_id];
6475
6476                 if (++level > max_level)
6477                         goto out;
6478
6479                 raw = &crtc_state->wm.g4x.raw[level];
6480                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6481                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6482                 raw->plane[PLANE_SPRITE0] = 0;
6483                 raw->fbc = active->sr.fbc;
6484
6485                 if (++level > max_level)
6486                         goto out;
6487
6488                 raw = &crtc_state->wm.g4x.raw[level];
6489                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6490                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6491                 raw->plane[PLANE_SPRITE0] = 0;
6492                 raw->fbc = active->hpll.fbc;
6493
6494         out:
6495                 for_each_plane_id_on_crtc(crtc, plane_id)
6496                         g4x_raw_plane_wm_set(crtc_state, level,
6497                                              plane_id, USHRT_MAX);
6498                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6499
6500                 crtc_state->wm.g4x.optimal = *active;
6501                 crtc_state->wm.g4x.intermediate = *active;
6502
6503                 drm_dbg_kms(&dev_priv->drm,
6504                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6505                             pipe_name(pipe),
6506                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6507                             wm->pipe[pipe].plane[PLANE_CURSOR],
6508                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
6509         }
6510
6511         drm_dbg_kms(&dev_priv->drm,
6512                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6513                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6514         drm_dbg_kms(&dev_priv->drm,
6515                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6516                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6517         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6518                     yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6519 }
6520
6521 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6522 {
6523         struct intel_plane *plane;
6524         struct intel_crtc *crtc;
6525
6526         mutex_lock(&dev_priv->wm.wm_mutex);
6527
6528         for_each_intel_plane(&dev_priv->drm, plane) {
6529                 struct intel_crtc *crtc =
6530                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6531                 struct intel_crtc_state *crtc_state =
6532                         to_intel_crtc_state(crtc->base.state);
6533                 struct intel_plane_state *plane_state =
6534                         to_intel_plane_state(plane->base.state);
6535                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6536                 enum plane_id plane_id = plane->id;
6537                 int level;
6538
6539                 if (plane_state->uapi.visible)
6540                         continue;
6541
6542                 for (level = 0; level < 3; level++) {
6543                         struct g4x_pipe_wm *raw =
6544                                 &crtc_state->wm.g4x.raw[level];
6545
6546                         raw->plane[plane_id] = 0;
6547                         wm_state->wm.plane[plane_id] = 0;
6548                 }
6549
6550                 if (plane_id == PLANE_PRIMARY) {
6551                         for (level = 0; level < 3; level++) {
6552                                 struct g4x_pipe_wm *raw =
6553                                         &crtc_state->wm.g4x.raw[level];
6554                                 raw->fbc = 0;
6555                         }
6556
6557                         wm_state->sr.fbc = 0;
6558                         wm_state->hpll.fbc = 0;
6559                         wm_state->fbc_en = false;
6560                 }
6561         }
6562
6563         for_each_intel_crtc(&dev_priv->drm, crtc) {
6564                 struct intel_crtc_state *crtc_state =
6565                         to_intel_crtc_state(crtc->base.state);
6566
6567                 crtc_state->wm.g4x.intermediate =
6568                         crtc_state->wm.g4x.optimal;
6569                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6570         }
6571
6572         g4x_program_watermarks(dev_priv);
6573
6574         mutex_unlock(&dev_priv->wm.wm_mutex);
6575 }
6576
6577 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6578 {
6579         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6580         struct intel_crtc *crtc;
6581         u32 val;
6582
6583         vlv_read_wm_values(dev_priv, wm);
6584
6585         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6586         wm->level = VLV_WM_LEVEL_PM2;
6587
6588         if (IS_CHERRYVIEW(dev_priv)) {
6589                 vlv_punit_get(dev_priv);
6590
6591                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6592                 if (val & DSP_MAXFIFO_PM5_ENABLE)
6593                         wm->level = VLV_WM_LEVEL_PM5;
6594
6595                 /*
6596                  * If DDR DVFS is disabled in the BIOS, Punit
6597                  * will never ack the request. So if that happens
6598                  * assume we don't have to enable/disable DDR DVFS
6599                  * dynamically. To test that just set the REQ_ACK
6600                  * bit to poke the Punit, but don't change the
6601                  * HIGH/LOW bits so that we don't actually change
6602                  * the current state.
6603                  */
6604                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6605                 val |= FORCE_DDR_FREQ_REQ_ACK;
6606                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6607
6608                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6609                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6610                         drm_dbg_kms(&dev_priv->drm,
6611                                     "Punit not acking DDR DVFS request, "
6612                                     "assuming DDR DVFS is disabled\n");
6613                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6614                 } else {
6615                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6616                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6617                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6618                 }
6619
6620                 vlv_punit_put(dev_priv);
6621         }
6622
6623         for_each_intel_crtc(&dev_priv->drm, crtc) {
6624                 struct intel_crtc_state *crtc_state =
6625                         to_intel_crtc_state(crtc->base.state);
6626                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6627                 const struct vlv_fifo_state *fifo_state =
6628                         &crtc_state->wm.vlv.fifo_state;
6629                 enum pipe pipe = crtc->pipe;
6630                 enum plane_id plane_id;
6631                 int level;
6632
6633                 vlv_get_fifo_size(crtc_state);
6634
6635                 active->num_levels = wm->level + 1;
6636                 active->cxsr = wm->cxsr;
6637
6638                 for (level = 0; level < active->num_levels; level++) {
6639                         struct g4x_pipe_wm *raw =
6640                                 &crtc_state->wm.vlv.raw[level];
6641
6642                         active->sr[level].plane = wm->sr.plane;
6643                         active->sr[level].cursor = wm->sr.cursor;
6644
6645                         for_each_plane_id_on_crtc(crtc, plane_id) {
6646                                 active->wm[level].plane[plane_id] =
6647                                         wm->pipe[pipe].plane[plane_id];
6648
6649                                 raw->plane[plane_id] =
6650                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
6651                                                             fifo_state->plane[plane_id]);
6652                         }
6653                 }
6654
6655                 for_each_plane_id_on_crtc(crtc, plane_id)
6656                         vlv_raw_plane_wm_set(crtc_state, level,
6657                                              plane_id, USHRT_MAX);
6658                 vlv_invalidate_wms(crtc, active, level);
6659
6660                 crtc_state->wm.vlv.optimal = *active;
6661                 crtc_state->wm.vlv.intermediate = *active;
6662
6663                 drm_dbg_kms(&dev_priv->drm,
6664                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6665                             pipe_name(pipe),
6666                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6667                             wm->pipe[pipe].plane[PLANE_CURSOR],
6668                             wm->pipe[pipe].plane[PLANE_SPRITE0],
6669                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
6670         }
6671
6672         drm_dbg_kms(&dev_priv->drm,
6673                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6674                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6675 }
6676
6677 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6678 {
6679         struct intel_plane *plane;
6680         struct intel_crtc *crtc;
6681
6682         mutex_lock(&dev_priv->wm.wm_mutex);
6683
6684         for_each_intel_plane(&dev_priv->drm, plane) {
6685                 struct intel_crtc *crtc =
6686                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6687                 struct intel_crtc_state *crtc_state =
6688                         to_intel_crtc_state(crtc->base.state);
6689                 struct intel_plane_state *plane_state =
6690                         to_intel_plane_state(plane->base.state);
6691                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6692                 const struct vlv_fifo_state *fifo_state =
6693                         &crtc_state->wm.vlv.fifo_state;
6694                 enum plane_id plane_id = plane->id;
6695                 int level;
6696
6697                 if (plane_state->uapi.visible)
6698                         continue;
6699
6700                 for (level = 0; level < wm_state->num_levels; level++) {
6701                         struct g4x_pipe_wm *raw =
6702                                 &crtc_state->wm.vlv.raw[level];
6703
6704                         raw->plane[plane_id] = 0;
6705
6706                         wm_state->wm[level].plane[plane_id] =
6707                                 vlv_invert_wm_value(raw->plane[plane_id],
6708                                                     fifo_state->plane[plane_id]);
6709                 }
6710         }
6711
6712         for_each_intel_crtc(&dev_priv->drm, crtc) {
6713                 struct intel_crtc_state *crtc_state =
6714                         to_intel_crtc_state(crtc->base.state);
6715
6716                 crtc_state->wm.vlv.intermediate =
6717                         crtc_state->wm.vlv.optimal;
6718                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6719         }
6720
6721         vlv_program_watermarks(dev_priv);
6722
6723         mutex_unlock(&dev_priv->wm.wm_mutex);
6724 }
6725
6726 /*
6727  * FIXME should probably kill this and improve
6728  * the real watermark readout/sanitation instead
6729  */
6730 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6731 {
6732         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6733         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6734         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6735
6736         /*
6737          * Don't touch WM1S_LP_EN here.
6738          * Doing so could cause underruns.
6739          */
6740 }
6741
6742 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6743 {
6744         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6745         struct intel_crtc *crtc;
6746
6747         ilk_init_lp_watermarks(dev_priv);
6748
6749         for_each_intel_crtc(&dev_priv->drm, crtc)
6750                 ilk_pipe_wm_get_hw_state(crtc);
6751
6752         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6753         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6754         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6755
6756         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6757         if (INTEL_GEN(dev_priv) >= 7) {
6758                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6759                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6760         }
6761
6762         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6763                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6764                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6765         else if (IS_IVYBRIDGE(dev_priv))
6766                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6767                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6768
6769         hw->enable_fbc_wm =
6770                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6771 }
6772
6773 /**
6774  * intel_update_watermarks - update FIFO watermark values based on current modes
6775  * @crtc: the #intel_crtc on which to compute the WM
6776  *
6777  * Calculate watermark values for the various WM regs based on current mode
6778  * and plane configuration.
6779  *
6780  * There are several cases to deal with here:
6781  *   - normal (i.e. non-self-refresh)
6782  *   - self-refresh (SR) mode
6783  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6784  *   - lines are small relative to FIFO size (buffer can hold more than 2
6785  *     lines), so need to account for TLB latency
6786  *
6787  *   The normal calculation is:
6788  *     watermark = dotclock * bytes per pixel * latency
6789  *   where latency is platform & configuration dependent (we assume pessimal
6790  *   values here).
6791  *
6792  *   The SR calculation is:
6793  *     watermark = (trunc(latency/line time)+1) * surface width *
6794  *       bytes per pixel
6795  *   where
6796  *     line time = htotal / dotclock
6797  *     surface width = hdisplay for normal plane and 64 for cursor
6798  *   and latency is assumed to be high, as above.
6799  *
6800  * The final value programmed to the register should always be rounded up,
6801  * and include an extra 2 entries to account for clock crossings.
6802  *
6803  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6804  * to set the non-SR watermarks to 8.
6805  */
6806 void intel_update_watermarks(struct intel_crtc *crtc)
6807 {
6808         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6809
6810         if (dev_priv->display.update_wm)
6811                 dev_priv->display.update_wm(crtc);
6812 }
6813
6814 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6815 {
6816         u32 val;
6817
6818         if (!HAS_IPC(dev_priv))
6819                 return;
6820
6821         val = I915_READ(DISP_ARB_CTL2);
6822
6823         if (dev_priv->ipc_enabled)
6824                 val |= DISP_IPC_ENABLE;
6825         else
6826                 val &= ~DISP_IPC_ENABLE;
6827
6828         I915_WRITE(DISP_ARB_CTL2, val);
6829 }
6830
6831 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6832 {
6833         /* Display WA #0477 WaDisableIPC: skl */
6834         if (IS_SKYLAKE(dev_priv))
6835                 return false;
6836
6837         /* Display WA #1141: SKL:all KBL:all CFL */
6838         if (IS_KABYLAKE(dev_priv) ||
6839             IS_COFFEELAKE(dev_priv) ||
6840             IS_COMETLAKE(dev_priv))
6841                 return dev_priv->dram_info.symmetric_memory;
6842
6843         return true;
6844 }
6845
6846 void intel_init_ipc(struct drm_i915_private *dev_priv)
6847 {
6848         if (!HAS_IPC(dev_priv))
6849                 return;
6850
6851         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6852
6853         intel_enable_ipc(dev_priv);
6854 }
6855
6856 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6857 {
6858         /*
6859          * On Ibex Peak and Cougar Point, we need to disable clock
6860          * gating for the panel power sequencer or it will fail to
6861          * start up when no ports are active.
6862          */
6863         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6864 }
6865
6866 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6867 {
6868         enum pipe pipe;
6869
6870         for_each_pipe(dev_priv, pipe) {
6871                 I915_WRITE(DSPCNTR(pipe),
6872                            I915_READ(DSPCNTR(pipe)) |
6873                            DISPPLANE_TRICKLE_FEED_DISABLE);
6874
6875                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6876                 POSTING_READ(DSPSURF(pipe));
6877         }
6878 }
6879
6880 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6881 {
6882         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6883
6884         /*
6885          * Required for FBC
6886          * WaFbcDisableDpfcClockGating:ilk
6887          */
6888         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6889                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6890                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6891
6892         I915_WRITE(PCH_3DCGDIS0,
6893                    MARIUNIT_CLOCK_GATE_DISABLE |
6894                    SVSMUNIT_CLOCK_GATE_DISABLE);
6895         I915_WRITE(PCH_3DCGDIS1,
6896                    VFMUNIT_CLOCK_GATE_DISABLE);
6897
6898         /*
6899          * According to the spec the following bits should be set in
6900          * order to enable memory self-refresh
6901          * The bit 22/21 of 0x42004
6902          * The bit 5 of 0x42020
6903          * The bit 15 of 0x45000
6904          */
6905         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6906                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
6907                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6908         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6909         I915_WRITE(DISP_ARB_CTL,
6910                    (I915_READ(DISP_ARB_CTL) |
6911                     DISP_FBC_WM_DIS));
6912
6913         /*
6914          * Based on the document from hardware guys the following bits
6915          * should be set unconditionally in order to enable FBC.
6916          * The bit 22 of 0x42000
6917          * The bit 22 of 0x42004
6918          * The bit 7,8,9 of 0x42020.
6919          */
6920         if (IS_IRONLAKE_M(dev_priv)) {
6921                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6922                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6923                            I915_READ(ILK_DISPLAY_CHICKEN1) |
6924                            ILK_FBCQ_DIS);
6925                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6926                            I915_READ(ILK_DISPLAY_CHICKEN2) |
6927                            ILK_DPARB_GATE);
6928         }
6929
6930         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6931
6932         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6933                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6934                    ILK_ELPIN_409_SELECT);
6935
6936         g4x_disable_trickle_feed(dev_priv);
6937
6938         ibx_init_clock_gating(dev_priv);
6939 }
6940
6941 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6942 {
6943         enum pipe pipe;
6944         u32 val;
6945
6946         /*
6947          * On Ibex Peak and Cougar Point, we need to disable clock
6948          * gating for the panel power sequencer or it will fail to
6949          * start up when no ports are active.
6950          */
6951         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6952                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6953                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6954         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6955                    DPLS_EDP_PPS_FIX_DIS);
6956         /* The below fixes the weird display corruption, a few pixels shifted
6957          * downward, on (only) LVDS of some HP laptops with IVY.
6958          */
6959         for_each_pipe(dev_priv, pipe) {
6960                 val = I915_READ(TRANS_CHICKEN2(pipe));
6961                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6962                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6963                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6964                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6965                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6966                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6967                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6968         }
6969         /* WADP0ClockGatingDisable */
6970         for_each_pipe(dev_priv, pipe) {
6971                 I915_WRITE(TRANS_CHICKEN1(pipe),
6972                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6973         }
6974 }
6975
6976 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6977 {
6978         u32 tmp;
6979
6980         tmp = I915_READ(MCH_SSKPD);
6981         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6982                 drm_dbg_kms(&dev_priv->drm,
6983                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6984                             tmp);
6985 }
6986
6987 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6988 {
6989         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6990
6991         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6992
6993         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6994                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6995                    ILK_ELPIN_409_SELECT);
6996
6997         I915_WRITE(GEN6_UCGCTL1,
6998                    I915_READ(GEN6_UCGCTL1) |
6999                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7000                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7001
7002         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7003          * gating disable must be set.  Failure to set it results in
7004          * flickering pixels due to Z write ordering failures after
7005          * some amount of runtime in the Mesa "fire" demo, and Unigine
7006          * Sanctuary and Tropics, and apparently anything else with
7007          * alpha test or pixel discard.
7008          *
7009          * According to the spec, bit 11 (RCCUNIT) must also be set,
7010          * but we didn't debug actual testcases to find it out.
7011          *
7012          * WaDisableRCCUnitClockGating:snb
7013          * WaDisableRCPBUnitClockGating:snb
7014          */
7015         I915_WRITE(GEN6_UCGCTL2,
7016                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7017                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7018
7019         /*
7020          * According to the spec the following bits should be
7021          * set in order to enable memory self-refresh and fbc:
7022          * The bit21 and bit22 of 0x42000
7023          * The bit21 and bit22 of 0x42004
7024          * The bit5 and bit7 of 0x42020
7025          * The bit14 of 0x70180
7026          * The bit14 of 0x71180
7027          *
7028          * WaFbcAsynchFlipDisableFbcQueue:snb
7029          */
7030         I915_WRITE(ILK_DISPLAY_CHICKEN1,
7031                    I915_READ(ILK_DISPLAY_CHICKEN1) |
7032                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7033         I915_WRITE(ILK_DISPLAY_CHICKEN2,
7034                    I915_READ(ILK_DISPLAY_CHICKEN2) |
7035                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7036         I915_WRITE(ILK_DSPCLK_GATE_D,
7037                    I915_READ(ILK_DSPCLK_GATE_D) |
7038                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
7039                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7040
7041         g4x_disable_trickle_feed(dev_priv);
7042
7043         cpt_init_clock_gating(dev_priv);
7044
7045         gen6_check_mch_setup(dev_priv);
7046 }
7047
7048 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7049 {
7050         /*
7051          * TODO: this bit should only be enabled when really needed, then
7052          * disabled when not needed anymore in order to save power.
7053          */
7054         if (HAS_PCH_LPT_LP(dev_priv))
7055                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7056                            I915_READ(SOUTH_DSPCLK_GATE_D) |
7057                            PCH_LP_PARTITION_LEVEL_DISABLE);
7058
7059         /* WADPOClockGatingDisable:hsw */
7060         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7061                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7062                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7063 }
7064
7065 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7066 {
7067         if (HAS_PCH_LPT_LP(dev_priv)) {
7068                 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7069
7070                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7071                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7072         }
7073 }
7074
7075 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7076                                    int general_prio_credits,
7077                                    int high_prio_credits)
7078 {
7079         u32 misccpctl;
7080         u32 val;
7081
7082         /* WaTempDisableDOPClkGating:bdw */
7083         misccpctl = I915_READ(GEN7_MISCCPCTL);
7084         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7085
7086         val = I915_READ(GEN8_L3SQCREG1);
7087         val &= ~L3_PRIO_CREDITS_MASK;
7088         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7089         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7090         I915_WRITE(GEN8_L3SQCREG1, val);
7091
7092         /*
7093          * Wait at least 100 clocks before re-enabling clock gating.
7094          * See the definition of L3SQCREG1 in BSpec.
7095          */
7096         POSTING_READ(GEN8_L3SQCREG1);
7097         udelay(1);
7098         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7099 }
7100
7101 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7102 {
7103         /* Wa_1409120013:icl,ehl */
7104         I915_WRITE(ILK_DPFC_CHICKEN,
7105                    ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7106
7107         /* This is not an Wa. Enable to reduce Sampler power */
7108         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7109                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7110
7111         /*Wa_14010594013:icl, ehl */
7112         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7113                          0, CNL_DELAY_PMRSP);
7114 }
7115
7116 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7117 {
7118         u32 vd_pg_enable = 0;
7119         unsigned int i;
7120
7121         /* Wa_1409120013:tgl */
7122         I915_WRITE(ILK_DPFC_CHICKEN,
7123                    ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7124
7125         /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7126         for (i = 0; i < I915_MAX_VCS; i++) {
7127                 if (HAS_ENGINE(&dev_priv->gt, _VCS(i)))
7128                         vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7129                                         VDN_MFX_POWERGATE_ENABLE(i);
7130         }
7131
7132         I915_WRITE(POWERGATE_ENABLE,
7133                    I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7134
7135         /* Wa_1409825376:tgl (pre-prod)*/
7136         if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
7137                 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7138                            TGL_VRH_GATING_DIS);
7139
7140         /* Wa_14011059788:tgl */
7141         intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7142                          0, DFR_DISABLE);
7143 }
7144
7145 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7146 {
7147         if (!HAS_PCH_CNP(dev_priv))
7148                 return;
7149
7150         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7151         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7152                    CNP_PWM_CGE_GATING_DISABLE);
7153 }
7154
7155 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7156 {
7157         u32 val;
7158         cnp_init_clock_gating(dev_priv);
7159
7160         /* This is not an Wa. Enable for better image quality */
7161         I915_WRITE(_3D_CHICKEN3,
7162                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7163
7164         /* WaEnableChickenDCPR:cnl */
7165         I915_WRITE(GEN8_CHICKEN_DCPR_1,
7166                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7167
7168         /* WaFbcWakeMemOn:cnl */
7169         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7170                    DISP_FBC_MEMORY_WAKE);
7171
7172         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7173         /* ReadHitWriteOnlyDisable:cnl */
7174         val |= RCCUNIT_CLKGATE_DIS;
7175         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7176
7177         /* Wa_2201832410:cnl */
7178         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7179         val |= GWUNIT_CLKGATE_DIS;
7180         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7181
7182         /* WaDisableVFclkgate:cnl */
7183         /* WaVFUnitClockGatingDisable:cnl */
7184         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7185         val |= VFUNIT_CLKGATE_DIS;
7186         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7187 }
7188
7189 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7190 {
7191         cnp_init_clock_gating(dev_priv);
7192         gen9_init_clock_gating(dev_priv);
7193
7194         /* WaFbcTurnOffFbcWatermark:cfl */
7195         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7196                    DISP_FBC_WM_DIS);
7197
7198         /* WaFbcNukeOnHostModify:cfl */
7199         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7200                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7201 }
7202
7203 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7204 {
7205         gen9_init_clock_gating(dev_priv);
7206
7207         /* WaDisableSDEUnitClockGating:kbl */
7208         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7209                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7210                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7211
7212         /* WaDisableGamClockGating:kbl */
7213         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7214                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7215                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7216
7217         /* WaFbcTurnOffFbcWatermark:kbl */
7218         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7219                    DISP_FBC_WM_DIS);
7220
7221         /* WaFbcNukeOnHostModify:kbl */
7222         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7223                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7224 }
7225
7226 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7227 {
7228         gen9_init_clock_gating(dev_priv);
7229
7230         /* WAC6entrylatency:skl */
7231         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7232                    FBC_LLC_FULLY_OPEN);
7233
7234         /* WaFbcTurnOffFbcWatermark:skl */
7235         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7236                    DISP_FBC_WM_DIS);
7237
7238         /* WaFbcNukeOnHostModify:skl */
7239         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7240                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7241
7242         /* WaFbcHighMemBwCorruptionAvoidance:skl */
7243         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7244                    ILK_DPFC_DISABLE_DUMMY0);
7245 }
7246
7247 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7248 {
7249         enum pipe pipe;
7250
7251         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7252         I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7253                    I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7254                    HSW_FBCQ_DIS);
7255
7256         /* WaSwitchSolVfFArbitrationPriority:bdw */
7257         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7258
7259         /* WaPsrDPAMaskVBlankInSRD:bdw */
7260         I915_WRITE(CHICKEN_PAR1_1,
7261                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7262
7263         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7264         for_each_pipe(dev_priv, pipe) {
7265                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7266                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
7267                            BDW_DPRS_MASK_VBLANK_SRD);
7268         }
7269
7270         /* WaVSRefCountFullforceMissDisable:bdw */
7271         /* WaDSRefCountFullforceMissDisable:bdw */
7272         I915_WRITE(GEN7_FF_THREAD_MODE,
7273                    I915_READ(GEN7_FF_THREAD_MODE) &
7274                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7275
7276         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7277                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7278
7279         /* WaDisableSDEUnitClockGating:bdw */
7280         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7281                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7282
7283         /* WaProgramL3SqcReg1Default:bdw */
7284         gen8_set_l3sqc_credits(dev_priv, 30, 2);
7285
7286         /* WaKVMNotificationOnConfigChange:bdw */
7287         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7288                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7289
7290         lpt_init_clock_gating(dev_priv);
7291
7292         /* WaDisableDopClockGating:bdw
7293          *
7294          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7295          * clock gating.
7296          */
7297         I915_WRITE(GEN6_UCGCTL1,
7298                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7299 }
7300
7301 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7302 {
7303         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7304         I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7305                    I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7306                    HSW_FBCQ_DIS);
7307
7308         /* This is required by WaCatErrorRejectionIssue:hsw */
7309         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7310                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7311                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7312
7313         /* WaSwitchSolVfFArbitrationPriority:hsw */
7314         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7315
7316         lpt_init_clock_gating(dev_priv);
7317 }
7318
7319 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7320 {
7321         u32 snpcr;
7322
7323         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7324
7325         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7326         I915_WRITE(ILK_DISPLAY_CHICKEN1,
7327                    I915_READ(ILK_DISPLAY_CHICKEN1) |
7328                    ILK_FBCQ_DIS);
7329
7330         /* WaDisableBackToBackFlipFix:ivb */
7331         I915_WRITE(IVB_CHICKEN3,
7332                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7333                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7334
7335         if (IS_IVB_GT1(dev_priv))
7336                 I915_WRITE(GEN7_ROW_CHICKEN2,
7337                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7338         else {
7339                 /* must write both registers */
7340                 I915_WRITE(GEN7_ROW_CHICKEN2,
7341                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7342                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7343                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7344         }
7345
7346         /*
7347          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7348          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7349          */
7350         I915_WRITE(GEN6_UCGCTL2,
7351                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7352
7353         /* This is required by WaCatErrorRejectionIssue:ivb */
7354         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7355                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7356                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7357
7358         g4x_disable_trickle_feed(dev_priv);
7359
7360         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7361         snpcr &= ~GEN6_MBC_SNPCR_MASK;
7362         snpcr |= GEN6_MBC_SNPCR_MED;
7363         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7364
7365         if (!HAS_PCH_NOP(dev_priv))
7366                 cpt_init_clock_gating(dev_priv);
7367
7368         gen6_check_mch_setup(dev_priv);
7369 }
7370
7371 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7372 {
7373         /* WaDisableBackToBackFlipFix:vlv */
7374         I915_WRITE(IVB_CHICKEN3,
7375                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7376                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7377
7378         /* WaDisableDopClockGating:vlv */
7379         I915_WRITE(GEN7_ROW_CHICKEN2,
7380                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7381
7382         /* This is required by WaCatErrorRejectionIssue:vlv */
7383         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7384                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7385                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7386
7387         /*
7388          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7389          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7390          */
7391         I915_WRITE(GEN6_UCGCTL2,
7392                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7393
7394         /* WaDisableL3Bank2xClockGate:vlv
7395          * Disabling L3 clock gating- MMIO 940c[25] = 1
7396          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7397         I915_WRITE(GEN7_UCGCTL4,
7398                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7399
7400         /*
7401          * WaDisableVLVClockGating_VBIIssue:vlv
7402          * Disable clock gating on th GCFG unit to prevent a delay
7403          * in the reporting of vblank events.
7404          */
7405         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7406 }
7407
7408 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7409 {
7410         /* WaVSRefCountFullforceMissDisable:chv */
7411         /* WaDSRefCountFullforceMissDisable:chv */
7412         I915_WRITE(GEN7_FF_THREAD_MODE,
7413                    I915_READ(GEN7_FF_THREAD_MODE) &
7414                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7415
7416         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7417         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7418                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7419
7420         /* WaDisableCSUnitClockGating:chv */
7421         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7422                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7423
7424         /* WaDisableSDEUnitClockGating:chv */
7425         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7426                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7427
7428         /*
7429          * WaProgramL3SqcReg1Default:chv
7430          * See gfxspecs/Related Documents/Performance Guide/
7431          * LSQC Setting Recommendations.
7432          */
7433         gen8_set_l3sqc_credits(dev_priv, 38, 2);
7434 }
7435
7436 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7437 {
7438         u32 dspclk_gate;
7439
7440         I915_WRITE(RENCLK_GATE_D1, 0);
7441         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7442                    GS_UNIT_CLOCK_GATE_DISABLE |
7443                    CL_UNIT_CLOCK_GATE_DISABLE);
7444         I915_WRITE(RAMCLK_GATE_D, 0);
7445         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7446                 OVRUNIT_CLOCK_GATE_DISABLE |
7447                 OVCUNIT_CLOCK_GATE_DISABLE;
7448         if (IS_GM45(dev_priv))
7449                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7450         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7451
7452         g4x_disable_trickle_feed(dev_priv);
7453 }
7454
7455 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7456 {
7457         struct intel_uncore *uncore = &dev_priv->uncore;
7458
7459         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7460         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7461         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7462         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7463         intel_uncore_write16(uncore, DEUC, 0);
7464         intel_uncore_write(uncore,
7465                            MI_ARB_STATE,
7466                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7467 }
7468
7469 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7470 {
7471         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7472                    I965_RCC_CLOCK_GATE_DISABLE |
7473                    I965_RCPB_CLOCK_GATE_DISABLE |
7474                    I965_ISC_CLOCK_GATE_DISABLE |
7475                    I965_FBC_CLOCK_GATE_DISABLE);
7476         I915_WRITE(RENCLK_GATE_D2, 0);
7477         I915_WRITE(MI_ARB_STATE,
7478                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7479 }
7480
7481 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7482 {
7483         u32 dstate = I915_READ(D_STATE);
7484
7485         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7486                 DSTATE_DOT_CLOCK_GATING;
7487         I915_WRITE(D_STATE, dstate);
7488
7489         if (IS_PINEVIEW(dev_priv))
7490                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7491
7492         /* IIR "flip pending" means done if this bit is set */
7493         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7494
7495         /* interrupts should cause a wake up from C3 */
7496         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7497
7498         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7499         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7500
7501         I915_WRITE(MI_ARB_STATE,
7502                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7503 }
7504
7505 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7506 {
7507         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7508
7509         /* interrupts should cause a wake up from C3 */
7510         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7511                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7512
7513         I915_WRITE(MEM_MODE,
7514                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7515
7516         /*
7517          * Have FBC ignore 3D activity since we use software
7518          * render tracking, and otherwise a pure 3D workload
7519          * (even if it just renders a single frame and then does
7520          * abosultely nothing) would not allow FBC to recompress
7521          * until a 2D blit occurs.
7522          */
7523         I915_WRITE(SCPD0,
7524                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7525 }
7526
7527 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7528 {
7529         I915_WRITE(MEM_MODE,
7530                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7531                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7532 }
7533
7534 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7535 {
7536         dev_priv->display.init_clock_gating(dev_priv);
7537 }
7538
7539 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7540 {
7541         if (HAS_PCH_LPT(dev_priv))
7542                 lpt_suspend_hw(dev_priv);
7543 }
7544
7545 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7546 {
7547         drm_dbg_kms(&dev_priv->drm,
7548                     "No clock gating settings or workarounds applied.\n");
7549 }
7550
7551 /**
7552  * intel_init_clock_gating_hooks - setup the clock gating hooks
7553  * @dev_priv: device private
7554  *
7555  * Setup the hooks that configure which clocks of a given platform can be
7556  * gated and also apply various GT and display specific workarounds for these
7557  * platforms. Note that some GT specific workarounds are applied separately
7558  * when GPU contexts or batchbuffers start their execution.
7559  */
7560 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7561 {
7562         if (IS_GEN(dev_priv, 12))
7563                 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7564         else if (IS_GEN(dev_priv, 11))
7565                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7566         else if (IS_CANNONLAKE(dev_priv))
7567                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7568         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7569                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7570         else if (IS_SKYLAKE(dev_priv))
7571                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7572         else if (IS_KABYLAKE(dev_priv))
7573                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7574         else if (IS_BROXTON(dev_priv))
7575                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7576         else if (IS_GEMINILAKE(dev_priv))
7577                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7578         else if (IS_BROADWELL(dev_priv))
7579                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7580         else if (IS_CHERRYVIEW(dev_priv))
7581                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7582         else if (IS_HASWELL(dev_priv))
7583                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7584         else if (IS_IVYBRIDGE(dev_priv))
7585                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7586         else if (IS_VALLEYVIEW(dev_priv))
7587                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7588         else if (IS_GEN(dev_priv, 6))
7589                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7590         else if (IS_GEN(dev_priv, 5))
7591                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7592         else if (IS_G4X(dev_priv))
7593                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7594         else if (IS_I965GM(dev_priv))
7595                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7596         else if (IS_I965G(dev_priv))
7597                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7598         else if (IS_GEN(dev_priv, 3))
7599                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7600         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7601                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7602         else if (IS_GEN(dev_priv, 2))
7603                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7604         else {
7605                 MISSING_CASE(INTEL_DEVID(dev_priv));
7606                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7607         }
7608 }
7609
7610 /* Set up chip specific power management-related functions */
7611 void intel_init_pm(struct drm_i915_private *dev_priv)
7612 {
7613         /* For cxsr */
7614         if (IS_PINEVIEW(dev_priv))
7615                 pnv_get_mem_freq(dev_priv);
7616         else if (IS_GEN(dev_priv, 5))
7617                 ilk_get_mem_freq(dev_priv);
7618
7619         if (intel_has_sagv(dev_priv))
7620                 skl_setup_sagv_block_time(dev_priv);
7621
7622         /* For FIFO watermark updates */
7623         if (INTEL_GEN(dev_priv) >= 9) {
7624                 skl_setup_wm_latency(dev_priv);
7625                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7626         } else if (HAS_PCH_SPLIT(dev_priv)) {
7627                 ilk_setup_wm_latency(dev_priv);
7628
7629                 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7630                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7631                     (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7632                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7633                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7634                         dev_priv->display.compute_intermediate_wm =
7635                                 ilk_compute_intermediate_wm;
7636                         dev_priv->display.initial_watermarks =
7637                                 ilk_initial_watermarks;
7638                         dev_priv->display.optimize_watermarks =
7639                                 ilk_optimize_watermarks;
7640                 } else {
7641                         drm_dbg_kms(&dev_priv->drm,
7642                                     "Failed to read display plane latency. "
7643                                     "Disable CxSR\n");
7644                 }
7645         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7646                 vlv_setup_wm_latency(dev_priv);
7647                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7648                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7649                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7650                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7651                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7652         } else if (IS_G4X(dev_priv)) {
7653                 g4x_setup_wm_latency(dev_priv);
7654                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7655                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7656                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7657                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7658         } else if (IS_PINEVIEW(dev_priv)) {
7659                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7660                                             dev_priv->is_ddr3,
7661                                             dev_priv->fsb_freq,
7662                                             dev_priv->mem_freq)) {
7663                         drm_info(&dev_priv->drm,
7664                                  "failed to find known CxSR latency "
7665                                  "(found ddr%s fsb freq %d, mem freq %d), "
7666                                  "disabling CxSR\n",
7667                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7668                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7669                         /* Disable CxSR and never update its watermark again */
7670                         intel_set_memory_cxsr(dev_priv, false);
7671                         dev_priv->display.update_wm = NULL;
7672                 } else
7673                         dev_priv->display.update_wm = pnv_update_wm;
7674         } else if (IS_GEN(dev_priv, 4)) {
7675                 dev_priv->display.update_wm = i965_update_wm;
7676         } else if (IS_GEN(dev_priv, 3)) {
7677                 dev_priv->display.update_wm = i9xx_update_wm;
7678                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7679         } else if (IS_GEN(dev_priv, 2)) {
7680                 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7681                         dev_priv->display.update_wm = i845_update_wm;
7682                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7683                 } else {
7684                         dev_priv->display.update_wm = i9xx_update_wm;
7685                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7686                 }
7687         } else {
7688                 drm_err(&dev_priv->drm,
7689                         "unexpected fall-through in %s\n", __func__);
7690         }
7691 }
7692
7693 void intel_pm_setup(struct drm_i915_private *dev_priv)
7694 {
7695         dev_priv->runtime_pm.suspended = false;
7696         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7697 }
7698
7699 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7700 {
7701         struct intel_dbuf_state *dbuf_state;
7702
7703         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7704         if (!dbuf_state)
7705                 return NULL;
7706
7707         return &dbuf_state->base;
7708 }
7709
7710 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7711                                      struct intel_global_state *state)
7712 {
7713         kfree(state);
7714 }
7715
7716 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7717         .atomic_duplicate_state = intel_dbuf_duplicate_state,
7718         .atomic_destroy_state = intel_dbuf_destroy_state,
7719 };
7720
7721 struct intel_dbuf_state *
7722 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7723 {
7724         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7725         struct intel_global_state *dbuf_state;
7726
7727         dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7728         if (IS_ERR(dbuf_state))
7729                 return ERR_CAST(dbuf_state);
7730
7731         return to_intel_dbuf_state(dbuf_state);
7732 }
7733
7734 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7735 {
7736         struct intel_dbuf_state *dbuf_state;
7737
7738         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7739         if (!dbuf_state)
7740                 return -ENOMEM;
7741
7742         intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7743                                      &dbuf_state->base, &intel_dbuf_funcs);
7744
7745         return 0;
7746 }
7747
7748 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7749 {
7750         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7751         const struct intel_dbuf_state *new_dbuf_state =
7752                 intel_atomic_get_new_dbuf_state(state);
7753         const struct intel_dbuf_state *old_dbuf_state =
7754                 intel_atomic_get_old_dbuf_state(state);
7755
7756         if (!new_dbuf_state ||
7757             new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7758                 return;
7759
7760         WARN_ON(!new_dbuf_state->base.changed);
7761
7762         gen9_dbuf_slices_update(dev_priv,
7763                                 old_dbuf_state->enabled_slices |
7764                                 new_dbuf_state->enabled_slices);
7765 }
7766
7767 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7768 {
7769         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7770         const struct intel_dbuf_state *new_dbuf_state =
7771                 intel_atomic_get_new_dbuf_state(state);
7772         const struct intel_dbuf_state *old_dbuf_state =
7773                 intel_atomic_get_old_dbuf_state(state);
7774
7775         if (!new_dbuf_state ||
7776             new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7777                 return;
7778
7779         WARN_ON(!new_dbuf_state->base.changed);
7780
7781         gen9_dbuf_slices_update(dev_priv,
7782                                 new_dbuf_state->enabled_slices);
7783 }