drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include <drm/drm_atomic_helper.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include "display/intel_atomic.h"
  36 #include "display/intel_display_types.h"
  37 #include "display/intel_fbc.h"
  38 #include "display/intel_sprite.h"
  39
  40 #include "gt/intel_llc.h"
  41
  42 #include "i915_drv.h"
  43 #include "i915_fixed.h"
  44 #include "i915_irq.h"
  45 #include "i915_trace.h"
  46 #include "display/intel_bw.h"
  47 #include "intel_pm.h"
  48 #include "intel_sideband.h"
  49 #include "../../../platform/x86/intel_ips.h"
  50
  51 /* Stores plane specific WM parameters */
  52 struct skl_wm_params {
  53         bool x_tiled, y_tiled;
  54         bool rc_surface;
  55         bool is_planar;
  56         u32 width;
  57         u8 cpp;
  58         u32 plane_pixel_rate;
  59         u32 y_min_scanlines;
  60         u32 plane_bytes_per_line;
  61         uint_fixed_16_16_t plane_blocks_per_line;
  62         uint_fixed_16_16_t y_tile_minimum;
  63         u32 linetime_us;
  64         u32 dbuf_block_size;
  65 };
  66
  67 /* used in computing the new watermarks state */
  68 struct intel_wm_config {
  69         unsigned int num_pipes_active;
  70         bool sprites_enabled;
  71         bool sprites_scaled;
  72 };
  73
  74 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  75 {
  76         if (HAS_LLC(dev_priv)) {
  77                 /*
  78                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  79                  * Display WA #0390: skl,kbl
  80                  *
  81                  * Must match Sampler, Pixel Back End, and Media. See
  82                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  83                  */
  84                 I915_WRITE(CHICKEN_PAR1_1,
  85                            I915_READ(CHICKEN_PAR1_1) |
  86                            SKL_DE_COMPRESSED_HASH_MODE);
  87         }
  88
  89         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  90         I915_WRITE(CHICKEN_PAR1_1,
  91                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  92
  93         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  94         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  95                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  96
  97         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  98         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  99         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 100                    DISP_FBC_WM_DIS |
 101                    DISP_FBC_MEMORY_WAKE);
 102
 103         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
 104         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
 105                    ILK_DPFC_DISABLE_DUMMY0);
 106
 107         if (IS_SKYLAKE(dev_priv)) {
 108                 /* WaDisableDopClockGating */
 109                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
 110                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 111         }
 112 }
 113
 114 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 115 {
 116         gen9_init_clock_gating(dev_priv);
 117
 118         /* WaDisableSDEUnitClockGating:bxt */
 119         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 120                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 121
 122         /*
 123          * FIXME:
 124          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 125          */
 126         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 127                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 128
 129         /*
 130          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 131          * to stay fully on.
 132          */
 133         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 134                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 135
 136         /*
 137          * Lower the display internal timeout.
 138          * This is needed to avoid any hard hangs when DSI port PLL
 139          * is off and a MMIO access is attempted by any privilege
 140          * application, using batch buffers or any other means.
 141          */
 142         I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 143 }
 144
 145 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 146 {
 147         gen9_init_clock_gating(dev_priv);
 148
 149         /*
 150          * WaDisablePWMClockGating:glk
 151          * Backlight PWM may stop in the asserted state, causing backlight
 152          * to stay fully on.
 153          */
 154         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 155                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 156 }
 157
 158 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 159 {
 160         u32 tmp;
 161
 162         tmp = I915_READ(CLKCFG);
 163
 164         switch (tmp & CLKCFG_FSB_MASK) {
 165         case CLKCFG_FSB_533:
 166                 dev_priv->fsb_freq = 533; /* 133*4 */
 167                 break;
 168         case CLKCFG_FSB_800:
 169                 dev_priv->fsb_freq = 800; /* 200*4 */
 170                 break;
 171         case CLKCFG_FSB_667:
 172                 dev_priv->fsb_freq =  667; /* 167*4 */
 173                 break;
 174         case CLKCFG_FSB_400:
 175                 dev_priv->fsb_freq = 400; /* 100*4 */
 176                 break;
 177         }
 178
 179         switch (tmp & CLKCFG_MEM_MASK) {
 180         case CLKCFG_MEM_533:
 181                 dev_priv->mem_freq = 533;
 182                 break;
 183         case CLKCFG_MEM_667:
 184                 dev_priv->mem_freq = 667;
 185                 break;
 186         case CLKCFG_MEM_800:
 187                 dev_priv->mem_freq = 800;
 188                 break;
 189         }
 190
 191         /* detect pineview DDR3 setting */
 192         tmp = I915_READ(CSHRDDR3CTL);
 193         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 194 }
 195
 196 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 197 {
 198         u16 ddrpll, csipll;
 199
 200         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 201         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 202
 203         switch (ddrpll & 0xff) {
 204         case 0xc:
 205                 dev_priv->mem_freq = 800;
 206                 break;
 207         case 0x10:
 208                 dev_priv->mem_freq = 1066;
 209                 break;
 210         case 0x14:
 211                 dev_priv->mem_freq = 1333;
 212                 break;
 213         case 0x18:
 214                 dev_priv->mem_freq = 1600;
 215                 break;
 216         default:
 217                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 218                         ddrpll & 0xff);
 219                 dev_priv->mem_freq = 0;
 220                 break;
 221         }
 222
 223         switch (csipll & 0x3ff) {
 224         case 0x00c:
 225                 dev_priv->fsb_freq = 3200;
 226                 break;
 227         case 0x00e:
 228                 dev_priv->fsb_freq = 3733;
 229                 break;
 230         case 0x010:
 231                 dev_priv->fsb_freq = 4266;
 232                 break;
 233         case 0x012:
 234                 dev_priv->fsb_freq = 4800;
 235                 break;
 236         case 0x014:
 237                 dev_priv->fsb_freq = 5333;
 238                 break;
 239         case 0x016:
 240                 dev_priv->fsb_freq = 5866;
 241                 break;
 242         case 0x018:
 243                 dev_priv->fsb_freq = 6400;
 244                 break;
 245         default:
 246                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 247                         csipll & 0x3ff);
 248                 dev_priv->fsb_freq = 0;
 249                 break;
 250         }
 251 }
 252
 253 static const struct cxsr_latency cxsr_latency_table[] = {
 254         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 255         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 256         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 257         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 258         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 259
 260         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 261         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 262         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 263         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 264         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 265
 266         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 267         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 268         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 269         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 270         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 271
 272         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 273         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 274         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 275         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 276         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 277
 278         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 279         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 280         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 281         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 282         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 283
 284         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 285         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 286         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 287         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 288         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 289 };
 290
 291 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 292                                                          bool is_ddr3,
 293                                                          int fsb,
 294                                                          int mem)
 295 {
 296         const struct cxsr_latency *latency;
 297         int i;
 298
 299         if (fsb == 0 || mem == 0)
 300                 return NULL;
 301
 302         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 303                 latency = &cxsr_latency_table[i];
 304                 if (is_desktop == latency->is_desktop &&
 305                     is_ddr3 == latency->is_ddr3 &&
 306                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 307                         return latency;
 308         }
 309
 310         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 311
 312         return NULL;
 313 }
 314
 315 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 316 {
 317         u32 val;
 318
 319         vlv_punit_get(dev_priv);
 320
 321         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 322         if (enable)
 323                 val &= ~FORCE_DDR_HIGH_FREQ;
 324         else
 325                 val |= FORCE_DDR_HIGH_FREQ;
 326         val &= ~FORCE_DDR_LOW_FREQ;
 327         val |= FORCE_DDR_FREQ_REQ_ACK;
 328         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 329
 330         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 331                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 332                 drm_err(&dev_priv->drm,
 333                         "timed out waiting for Punit DDR DVFS request\n");
 334
 335         vlv_punit_put(dev_priv);
 336 }
 337
 338 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 339 {
 340         u32 val;
 341
 342         vlv_punit_get(dev_priv);
 343
 344         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 345         if (enable)
 346                 val |= DSP_MAXFIFO_PM5_ENABLE;
 347         else
 348                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 349         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 350
 351         vlv_punit_put(dev_priv);
 352 }
 353
 354 #define FW_WM(value, plane) \
 355         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 356
 357 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 358 {
 359         bool was_enabled;
 360         u32 val;
 361
 362         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 363                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 364                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 365                 POSTING_READ(FW_BLC_SELF_VLV);
 366         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 367                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 368                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 369                 POSTING_READ(FW_BLC_SELF);
 370         } else if (IS_PINEVIEW(dev_priv)) {
 371                 val = I915_READ(DSPFW3);
 372                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 373                 if (enable)
 374                         val |= PINEVIEW_SELF_REFRESH_EN;
 375                 else
 376                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 377                 I915_WRITE(DSPFW3, val);
 378                 POSTING_READ(DSPFW3);
 379         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 380                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 381                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 382                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 383                 I915_WRITE(FW_BLC_SELF, val);
 384                 POSTING_READ(FW_BLC_SELF);
 385         } else if (IS_I915GM(dev_priv)) {
 386                 /*
 387                  * FIXME can't find a bit like this for 915G, and
 388                  * and yet it does have the related watermark in
 389                  * FW_BLC_SELF. What's going on?
 390                  */
 391                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 392                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 393                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 394                 I915_WRITE(INSTPM, val);
 395                 POSTING_READ(INSTPM);
 396         } else {
 397                 return false;
 398         }
 399
 400         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 401
 402         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 403                     enableddisabled(enable),
 404                     enableddisabled(was_enabled));
 405
 406         return was_enabled;
 407 }
 408
 409 /**
 410  * intel_set_memory_cxsr - Configure CxSR state
 411  * @dev_priv: i915 device
 412  * @enable: Allow vs. disallow CxSR
 413  *
 414  * Allow or disallow the system to enter a special CxSR
 415  * (C-state self refresh) state. What typically happens in CxSR mode
 416  * is that several display FIFOs may get combined into a single larger
 417  * FIFO for a particular plane (so called max FIFO mode) to allow the
 418  * system to defer memory fetches longer, and the memory will enter
 419  * self refresh.
 420  *
 421  * Note that enabling CxSR does not guarantee that the system enter
 422  * this special mode, nor does it guarantee that the system stays
 423  * in that mode once entered. So this just allows/disallows the system
 424  * to autonomously utilize the CxSR mode. Other factors such as core
 425  * C-states will affect when/if the system actually enters/exits the
 426  * CxSR mode.
 427  *
 428  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 429  * and the system is free to enter/exit memory self refresh at any time
 430  * even when the use of CxSR has been disallowed.
 431  *
 432  * While the system is actually in the CxSR/max FIFO mode, some plane
 433  * control registers will not get latched on vblank. Thus in order to
 434  * guarantee the system will respond to changes in the plane registers
 435  * we must always disallow CxSR prior to making changes to those registers.
 436  * Unfortunately the system will re-evaluate the CxSR conditions at
 437  * frame start which happens after vblank start (which is when the plane
 438  * registers would get latched), so we can't proceed with the plane update
 439  * during the same frame where we disallowed CxSR.
 440  *
 441  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 442  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 443  * the hardware w.r.t. HPLL SR when writing to plane registers.
 444  * Disallowing just CxSR is sufficient.
 445  */
 446 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 447 {
 448         bool ret;
 449
 450         mutex_lock(&dev_priv->wm.wm_mutex);
 451         ret = _intel_set_memory_cxsr(dev_priv, enable);
 452         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 453                 dev_priv->wm.vlv.cxsr = enable;
 454         else if (IS_G4X(dev_priv))
 455                 dev_priv->wm.g4x.cxsr = enable;
 456         mutex_unlock(&dev_priv->wm.wm_mutex);
 457
 458         return ret;
 459 }
 460
 461 /*
 462  * Latency for FIFO fetches is dependent on several factors:
 463  *   - memory configuration (speed, channels)
 464  *   - chipset
 465  *   - current MCH state
 466  * It can be fairly high in some situations, so here we assume a fairly
 467  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 468  * set this value too high, the FIFO will fetch frequently to stay full)
 469  * and power consumption (set it too low to save power and we might see
 470  * FIFO underruns and display "flicker").
 471  *
 472  * A value of 5us seems to be a good balance; safe for very low end
 473  * platforms but not overly aggressive on lower latency configs.
 474  */
 475 static const int pessimal_latency_ns = 5000;
 476
 477 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 478         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 479
 480 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 481 {
 482         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 483         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 484         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 485         enum pipe pipe = crtc->pipe;
 486         int sprite0_start, sprite1_start;
 487         u32 dsparb, dsparb2, dsparb3;
 488
 489         switch (pipe) {
 490         case PIPE_A:
 491                 dsparb = I915_READ(DSPARB);
 492                 dsparb2 = I915_READ(DSPARB2);
 493                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 494                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 495                 break;
 496         case PIPE_B:
 497                 dsparb = I915_READ(DSPARB);
 498                 dsparb2 = I915_READ(DSPARB2);
 499                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 500                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 501                 break;
 502         case PIPE_C:
 503                 dsparb2 = I915_READ(DSPARB2);
 504                 dsparb3 = I915_READ(DSPARB3);
 505                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 506                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 507                 break;
 508         default:
 509                 MISSING_CASE(pipe);
 510                 return;
 511         }
 512
 513         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 514         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 515         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 516         fifo_state->plane[PLANE_CURSOR] = 63;
 517 }
 518
 519 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 520                               enum i9xx_plane_id i9xx_plane)
 521 {
 522         u32 dsparb = I915_READ(DSPARB);
 523         int size;
 524
 525         size = dsparb & 0x7f;
 526         if (i9xx_plane == PLANE_B)
 527                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 528
 529         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 530                     dsparb, plane_name(i9xx_plane), size);
 531
 532         return size;
 533 }
 534
 535 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 536                               enum i9xx_plane_id i9xx_plane)
 537 {
 538         u32 dsparb = I915_READ(DSPARB);
 539         int size;
 540
 541         size = dsparb & 0x1ff;
 542         if (i9xx_plane == PLANE_B)
 543                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 544         size >>= 1; /* Convert to cachelines */
 545
 546         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 547                     dsparb, plane_name(i9xx_plane), size);
 548
 549         return size;
 550 }
 551
 552 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 553                               enum i9xx_plane_id i9xx_plane)
 554 {
 555         u32 dsparb = I915_READ(DSPARB);
 556         int size;
 557
 558         size = dsparb & 0x7f;
 559         size >>= 2; /* Convert to cachelines */
 560
 561         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 562                     dsparb, plane_name(i9xx_plane), size);
 563
 564         return size;
 565 }
 566
 567 /* Pineview has different values for various configs */
 568 static const struct intel_watermark_params pnv_display_wm = {
 569         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 570         .max_wm = PINEVIEW_MAX_WM,
 571         .default_wm = PINEVIEW_DFT_WM,
 572         .guard_size = PINEVIEW_GUARD_WM,
 573         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 574 };
 575
 576 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 577         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 578         .max_wm = PINEVIEW_MAX_WM,
 579         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 580         .guard_size = PINEVIEW_GUARD_WM,
 581         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 582 };
 583
 584 static const struct intel_watermark_params pnv_cursor_wm = {
 585         .fifo_size = PINEVIEW_CURSOR_FIFO,
 586         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 587         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 588         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 589         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 590 };
 591
 592 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 593         .fifo_size = PINEVIEW_CURSOR_FIFO,
 594         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 595         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 596         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 597         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 598 };
 599
 600 static const struct intel_watermark_params i965_cursor_wm_info = {
 601         .fifo_size = I965_CURSOR_FIFO,
 602         .max_wm = I965_CURSOR_MAX_WM,
 603         .default_wm = I965_CURSOR_DFT_WM,
 604         .guard_size = 2,
 605         .cacheline_size = I915_FIFO_LINE_SIZE,
 606 };
 607
 608 static const struct intel_watermark_params i945_wm_info = {
 609         .fifo_size = I945_FIFO_SIZE,
 610         .max_wm = I915_MAX_WM,
 611         .default_wm = 1,
 612         .guard_size = 2,
 613         .cacheline_size = I915_FIFO_LINE_SIZE,
 614 };
 615
 616 static const struct intel_watermark_params i915_wm_info = {
 617         .fifo_size = I915_FIFO_SIZE,
 618         .max_wm = I915_MAX_WM,
 619         .default_wm = 1,
 620         .guard_size = 2,
 621         .cacheline_size = I915_FIFO_LINE_SIZE,
 622 };
 623
 624 static const struct intel_watermark_params i830_a_wm_info = {
 625         .fifo_size = I855GM_FIFO_SIZE,
 626         .max_wm = I915_MAX_WM,
 627         .default_wm = 1,
 628         .guard_size = 2,
 629         .cacheline_size = I830_FIFO_LINE_SIZE,
 630 };
 631
 632 static const struct intel_watermark_params i830_bc_wm_info = {
 633         .fifo_size = I855GM_FIFO_SIZE,
 634         .max_wm = I915_MAX_WM/2,
 635         .default_wm = 1,
 636         .guard_size = 2,
 637         .cacheline_size = I830_FIFO_LINE_SIZE,
 638 };
 639
 640 static const struct intel_watermark_params i845_wm_info = {
 641         .fifo_size = I830_FIFO_SIZE,
 642         .max_wm = I915_MAX_WM,
 643         .default_wm = 1,
 644         .guard_size = 2,
 645         .cacheline_size = I830_FIFO_LINE_SIZE,
 646 };
 647
 648 /**
 649  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 650  * @pixel_rate: Pipe pixel rate in kHz
 651  * @cpp: Plane bytes per pixel
 652  * @latency: Memory wakeup latency in 0.1us units
 653  *
 654  * Compute the watermark using the method 1 or "small buffer"
 655  * formula. The caller may additonally add extra cachelines
 656  * to account for TLB misses and clock crossings.
 657  *
 658  * This method is concerned with the short term drain rate
 659  * of the FIFO, ie. it does not account for blanking periods
 660  * which would effectively reduce the average drain rate across
 661  * a longer period. The name "small" refers to the fact the
 662  * FIFO is relatively small compared to the amount of data
 663  * fetched.
 664  *
 665  * The FIFO level vs. time graph might look something like:
 666  *
 667  *   |\   |\
 668  *   | \  | \
 669  * __---__---__ (- plane active, _ blanking)
 670  * -> time
 671  *
 672  * or perhaps like this:
 673  *
 674  *   |\|\  |\|\
 675  * __----__----__ (- plane active, _ blanking)
 676  * -> time
 677  *
 678  * Returns:
 679  * The watermark in bytes
 680  */
 681 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 682                                      unsigned int cpp,
 683                                      unsigned int latency)
 684 {
 685         u64 ret;
 686
 687         ret = mul_u32_u32(pixel_rate, cpp * latency);
 688         ret = DIV_ROUND_UP_ULL(ret, 10000);
 689
 690         return ret;
 691 }
 692
 693 /**
 694  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 695  * @pixel_rate: Pipe pixel rate in kHz
 696  * @htotal: Pipe horizontal total
 697  * @width: Plane width in pixels
 698  * @cpp: Plane bytes per pixel
 699  * @latency: Memory wakeup latency in 0.1us units
 700  *
 701  * Compute the watermark using the method 2 or "large buffer"
 702  * formula. The caller may additonally add extra cachelines
 703  * to account for TLB misses and clock crossings.
 704  *
 705  * This method is concerned with the long term drain rate
 706  * of the FIFO, ie. it does account for blanking periods
 707  * which effectively reduce the average drain rate across
 708  * a longer period. The name "large" refers to the fact the
 709  * FIFO is relatively large compared to the amount of data
 710  * fetched.
 711  *
 712  * The FIFO level vs. time graph might look something like:
 713  *
 714  *    |\___       |\___
 715  *    |    \___   |    \___
 716  *    |        \  |        \
 717  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 718  * -> time
 719  *
 720  * Returns:
 721  * The watermark in bytes
 722  */
 723 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 724                                      unsigned int htotal,
 725                                      unsigned int width,
 726                                      unsigned int cpp,
 727                                      unsigned int latency)
 728 {
 729         unsigned int ret;
 730
 731         /*
 732          * FIXME remove once all users are computing
 733          * watermarks in the correct place.
 734          */
 735         if (WARN_ON_ONCE(htotal == 0))
 736                 htotal = 1;
 737
 738         ret = (latency * pixel_rate) / (htotal * 10000);
 739         ret = (ret + 1) * width * cpp;
 740
 741         return ret;
 742 }
 743
 744 /**
 745  * intel_calculate_wm - calculate watermark level
 746  * @pixel_rate: pixel clock
 747  * @wm: chip FIFO params
 748  * @fifo_size: size of the FIFO buffer
 749  * @cpp: bytes per pixel
 750  * @latency_ns: memory latency for the platform
 751  *
 752  * Calculate the watermark level (the level at which the display plane will
 753  * start fetching from memory again).  Each chip has a different display
 754  * FIFO size and allocation, so the caller needs to figure that out and pass
 755  * in the correct intel_watermark_params structure.
 756  *
 757  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 758  * on the pixel size.  When it reaches the watermark level, it'll start
 759  * fetching FIFO line sized based chunks from memory until the FIFO fills
 760  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 761  * will occur, and a display engine hang could result.
 762  */
 763 static unsigned int intel_calculate_wm(int pixel_rate,
 764                                        const struct intel_watermark_params *wm,
 765                                        int fifo_size, int cpp,
 766                                        unsigned int latency_ns)
 767 {
 768         int entries, wm_size;
 769
 770         /*
 771          * Note: we need to make sure we don't overflow for various clock &
 772          * latency values.
 773          * clocks go from a few thousand to several hundred thousand.
 774          * latency is usually a few thousand
 775          */
 776         entries = intel_wm_method1(pixel_rate, cpp,
 777                                    latency_ns / 100);
 778         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 779                 wm->guard_size;
 780         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 781
 782         wm_size = fifo_size - entries;
 783         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 784
 785         /* Don't promote wm_size to unsigned... */
 786         if (wm_size > wm->max_wm)
 787                 wm_size = wm->max_wm;
 788         if (wm_size <= 0)
 789                 wm_size = wm->default_wm;
 790
 791         /*
 792          * Bspec seems to indicate that the value shouldn't be lower than
 793          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 794          * Lets go for 8 which is the burst size since certain platforms
 795          * already use a hardcoded 8 (which is what the spec says should be
 796          * done).
 797          */
 798         if (wm_size <= 8)
 799                 wm_size = 8;
 800
 801         return wm_size;
 802 }
 803
 804 static bool is_disabling(int old, int new, int threshold)
 805 {
 806         return old >= threshold && new < threshold;
 807 }
 808
 809 static bool is_enabling(int old, int new, int threshold)
 810 {
 811         return old < threshold && new >= threshold;
 812 }
 813
 814 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 815 {
 816         return dev_priv->wm.max_level + 1;
 817 }
 818
 819 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 820                                    const struct intel_plane_state *plane_state)
 821 {
 822         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 823
 824         /* FIXME check the 'enable' instead */
 825         if (!crtc_state->hw.active)
 826                 return false;
 827
 828         /*
 829          * Treat cursor with fb as always visible since cursor updates
 830          * can happen faster than the vrefresh rate, and the current
 831          * watermark code doesn't handle that correctly. Cursor updates
 832          * which set/clear the fb or change the cursor size are going
 833          * to get throttled by intel_legacy_cursor_update() to work
 834          * around this problem with the watermark code.
 835          */
 836         if (plane->id == PLANE_CURSOR)
 837                 return plane_state->hw.fb != NULL;
 838         else
 839                 return plane_state->uapi.visible;
 840 }
 841
 842 static bool intel_crtc_active(struct intel_crtc *crtc)
 843 {
 844         /* Be paranoid as we can arrive here with only partial
 845          * state retrieved from the hardware during setup.
 846          *
 847          * We can ditch the adjusted_mode.crtc_clock check as soon
 848          * as Haswell has gained clock readout/fastboot support.
 849          *
 850          * We can ditch the crtc->primary->state->fb check as soon as we can
 851          * properly reconstruct framebuffers.
 852          *
 853          * FIXME: The intel_crtc->active here should be switched to
 854          * crtc->state->active once we have proper CRTC states wired up
 855          * for atomic.
 856          */
 857         return crtc->active && crtc->base.primary->state->fb &&
 858                 crtc->config->hw.adjusted_mode.crtc_clock;
 859 }
 860
 861 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 862 {
 863         struct intel_crtc *crtc, *enabled = NULL;
 864
 865         for_each_intel_crtc(&dev_priv->drm, crtc) {
 866                 if (intel_crtc_active(crtc)) {
 867                         if (enabled)
 868                                 return NULL;
 869                         enabled = crtc;
 870                 }
 871         }
 872
 873         return enabled;
 874 }
 875
 876 static void pnv_update_wm(struct intel_crtc *unused_crtc)
 877 {
 878         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 879         struct intel_crtc *crtc;
 880         const struct cxsr_latency *latency;
 881         u32 reg;
 882         unsigned int wm;
 883
 884         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 885                                          dev_priv->is_ddr3,
 886                                          dev_priv->fsb_freq,
 887                                          dev_priv->mem_freq);
 888         if (!latency) {
 889                 drm_dbg_kms(&dev_priv->drm,
 890                             "Unknown FSB/MEM found, disable CxSR\n");
 891                 intel_set_memory_cxsr(dev_priv, false);
 892                 return;
 893         }
 894
 895         crtc = single_enabled_crtc(dev_priv);
 896         if (crtc) {
 897                 const struct drm_display_mode *adjusted_mode =
 898                         &crtc->config->hw.adjusted_mode;
 899                 const struct drm_framebuffer *fb =
 900                         crtc->base.primary->state->fb;
 901                 int cpp = fb->format->cpp[0];
 902                 int clock = adjusted_mode->crtc_clock;
 903
 904                 /* Display SR */
 905                 wm = intel_calculate_wm(clock, &pnv_display_wm,
 906                                         pnv_display_wm.fifo_size,
 907                                         cpp, latency->display_sr);
 908                 reg = I915_READ(DSPFW1);
 909                 reg &= ~DSPFW_SR_MASK;
 910                 reg |= FW_WM(wm, SR);
 911                 I915_WRITE(DSPFW1, reg);
 912                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 913
 914                 /* cursor SR */
 915                 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
 916                                         pnv_display_wm.fifo_size,
 917                                         4, latency->cursor_sr);
 918                 reg = I915_READ(DSPFW3);
 919                 reg &= ~DSPFW_CURSOR_SR_MASK;
 920                 reg |= FW_WM(wm, CURSOR_SR);
 921                 I915_WRITE(DSPFW3, reg);
 922
 923                 /* Display HPLL off SR */
 924                 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
 925                                         pnv_display_hplloff_wm.fifo_size,
 926                                         cpp, latency->display_hpll_disable);
 927                 reg = I915_READ(DSPFW3);
 928                 reg &= ~DSPFW_HPLL_SR_MASK;
 929                 reg |= FW_WM(wm, HPLL_SR);
 930                 I915_WRITE(DSPFW3, reg);
 931
 932                 /* cursor HPLL off SR */
 933                 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
 934                                         pnv_display_hplloff_wm.fifo_size,
 935                                         4, latency->cursor_hpll_disable);
 936                 reg = I915_READ(DSPFW3);
 937                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 938                 reg |= FW_WM(wm, HPLL_CURSOR);
 939                 I915_WRITE(DSPFW3, reg);
 940                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 941
 942                 intel_set_memory_cxsr(dev_priv, true);
 943         } else {
 944                 intel_set_memory_cxsr(dev_priv, false);
 945         }
 946 }
 947
 948 /*
 949  * Documentation says:
 950  * "If the line size is small, the TLB fetches can get in the way of the
 951  *  data fetches, causing some lag in the pixel data return which is not
 952  *  accounted for in the above formulas. The following adjustment only
 953  *  needs to be applied if eight whole lines fit in the buffer at once.
 954  *  The WM is adjusted upwards by the difference between the FIFO size
 955  *  and the size of 8 whole lines. This adjustment is always performed
 956  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 957  */
 958 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 959 {
 960         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 961
 962         return max(0, tlb_miss);
 963 }
 964
 965 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 966                                 const struct g4x_wm_values *wm)
 967 {
 968         enum pipe pipe;
 969
 970         for_each_pipe(dev_priv, pipe)
 971                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 972
 973         I915_WRITE(DSPFW1,
 974                    FW_WM(wm->sr.plane, SR) |
 975                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 976                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 977                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 978         I915_WRITE(DSPFW2,
 979                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 980                    FW_WM(wm->sr.fbc, FBC_SR) |
 981                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 982                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 983                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 984                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 985         I915_WRITE(DSPFW3,
 986                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 987                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 988                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 989                    FW_WM(wm->hpll.plane, HPLL_SR));
 990
 991         POSTING_READ(DSPFW1);
 992 }
 993
 994 #define FW_WM_VLV(value, plane) \
 995         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 996
 997 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 998                                 const struct vlv_wm_values *wm)
 999 {
1000         enum pipe pipe;
1001
1002         for_each_pipe(dev_priv, pipe) {
1003                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1004
1005                 I915_WRITE(VLV_DDL(pipe),
1006                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1007                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1008                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1009                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1010         }
1011
1012         /*
1013          * Zero the (unused) WM1 watermarks, and also clear all the
1014          * high order bits so that there are no out of bounds values
1015          * present in the registers during the reprogramming.
1016          */
1017         I915_WRITE(DSPHOWM, 0);
1018         I915_WRITE(DSPHOWM1, 0);
1019         I915_WRITE(DSPFW4, 0);
1020         I915_WRITE(DSPFW5, 0);
1021         I915_WRITE(DSPFW6, 0);
1022
1023         I915_WRITE(DSPFW1,
1024                    FW_WM(wm->sr.plane, SR) |
1025                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1026                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1027                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1028         I915_WRITE(DSPFW2,
1029                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1030                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1031                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1032         I915_WRITE(DSPFW3,
1033                    FW_WM(wm->sr.cursor, CURSOR_SR));
1034
1035         if (IS_CHERRYVIEW(dev_priv)) {
1036                 I915_WRITE(DSPFW7_CHV,
1037                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1038                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1039                 I915_WRITE(DSPFW8_CHV,
1040                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1041                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1042                 I915_WRITE(DSPFW9_CHV,
1043                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1044                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1045                 I915_WRITE(DSPHOWM,
1046                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1047                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1048                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1049                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1050                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1051                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1052                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1053                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1054                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1055                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1056         } else {
1057                 I915_WRITE(DSPFW7,
1058                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1059                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1060                 I915_WRITE(DSPHOWM,
1061                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1062                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1063                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1064                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1065                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1066                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1067                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1068         }
1069
1070         POSTING_READ(DSPFW1);
1071 }
1072
1073 #undef FW_WM_VLV
1074
1075 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1076 {
1077         /* all latencies in usec */
1078         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1079         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1080         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1081
1082         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1083 }
1084
1085 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1086 {
1087         /*
1088          * DSPCNTR[13] supposedly controls whether the
1089          * primary plane can use the FIFO space otherwise
1090          * reserved for the sprite plane. It's not 100% clear
1091          * what the actual FIFO size is, but it looks like we
1092          * can happily set both primary and sprite watermarks
1093          * up to 127 cachelines. So that would seem to mean
1094          * that either DSPCNTR[13] doesn't do anything, or that
1095          * the total FIFO is >= 256 cachelines in size. Either
1096          * way, we don't seem to have to worry about this
1097          * repartitioning as the maximum watermark value the
1098          * register can hold for each plane is lower than the
1099          * minimum FIFO size.
1100          */
1101         switch (plane_id) {
1102         case PLANE_CURSOR:
1103                 return 63;
1104         case PLANE_PRIMARY:
1105                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1106         case PLANE_SPRITE0:
1107                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1108         default:
1109                 MISSING_CASE(plane_id);
1110                 return 0;
1111         }
1112 }
1113
1114 static int g4x_fbc_fifo_size(int level)
1115 {
1116         switch (level) {
1117         case G4X_WM_LEVEL_SR:
1118                 return 7;
1119         case G4X_WM_LEVEL_HPLL:
1120                 return 15;
1121         default:
1122                 MISSING_CASE(level);
1123                 return 0;
1124         }
1125 }
1126
1127 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1128                           const struct intel_plane_state *plane_state,
1129                           int level)
1130 {
1131         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1132         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1133         const struct drm_display_mode *adjusted_mode =
1134                 &crtc_state->hw.adjusted_mode;
1135         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1136         unsigned int clock, htotal, cpp, width, wm;
1137
1138         if (latency == 0)
1139                 return USHRT_MAX;
1140
1141         if (!intel_wm_plane_visible(crtc_state, plane_state))
1142                 return 0;
1143
1144         cpp = plane_state->hw.fb->format->cpp[0];
1145
1146         /*
1147          * Not 100% sure which way ELK should go here as the
1148          * spec only says CL/CTG should assume 32bpp and BW
1149          * doesn't need to. But as these things followed the
1150          * mobile vs. desktop lines on gen3 as well, let's
1151          * assume ELK doesn't need this.
1152          *
1153          * The spec also fails to list such a restriction for
1154          * the HPLL watermark, which seems a little strange.
1155          * Let's use 32bpp for the HPLL watermark as well.
1156          */
1157         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1158             level != G4X_WM_LEVEL_NORMAL)
1159                 cpp = max(cpp, 4u);
1160
1161         clock = adjusted_mode->crtc_clock;
1162         htotal = adjusted_mode->crtc_htotal;
1163
1164         width = drm_rect_width(&plane_state->uapi.dst);
1165
1166         if (plane->id == PLANE_CURSOR) {
1167                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1168         } else if (plane->id == PLANE_PRIMARY &&
1169                    level == G4X_WM_LEVEL_NORMAL) {
1170                 wm = intel_wm_method1(clock, cpp, latency);
1171         } else {
1172                 unsigned int small, large;
1173
1174                 small = intel_wm_method1(clock, cpp, latency);
1175                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1176
1177                 wm = min(small, large);
1178         }
1179
1180         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1181                               width, cpp);
1182
1183         wm = DIV_ROUND_UP(wm, 64) + 2;
1184
1185         return min_t(unsigned int, wm, USHRT_MAX);
1186 }
1187
1188 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1189                                  int level, enum plane_id plane_id, u16 value)
1190 {
1191         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1192         bool dirty = false;
1193
1194         for (; level < intel_wm_num_levels(dev_priv); level++) {
1195                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1196
1197                 dirty |= raw->plane[plane_id] != value;
1198                 raw->plane[plane_id] = value;
1199         }
1200
1201         return dirty;
1202 }
1203
1204 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1205                                int level, u16 value)
1206 {
1207         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1208         bool dirty = false;
1209
1210         /* NORMAL level doesn't have an FBC watermark */
1211         level = max(level, G4X_WM_LEVEL_SR);
1212
1213         for (; level < intel_wm_num_levels(dev_priv); level++) {
1214                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1215
1216                 dirty |= raw->fbc != value;
1217                 raw->fbc = value;
1218         }
1219
1220         return dirty;
1221 }
1222
1223 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1224                               const struct intel_plane_state *plane_state,
1225                               u32 pri_val);
1226
1227 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1228                                      const struct intel_plane_state *plane_state)
1229 {
1230         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1231         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1232         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1233         enum plane_id plane_id = plane->id;
1234         bool dirty = false;
1235         int level;
1236
1237         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1238                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1239                 if (plane_id == PLANE_PRIMARY)
1240                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1241                 goto out;
1242         }
1243
1244         for (level = 0; level < num_levels; level++) {
1245                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1246                 int wm, max_wm;
1247
1248                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1249                 max_wm = g4x_plane_fifo_size(plane_id, level);
1250
1251                 if (wm > max_wm)
1252                         break;
1253
1254                 dirty |= raw->plane[plane_id] != wm;
1255                 raw->plane[plane_id] = wm;
1256
1257                 if (plane_id != PLANE_PRIMARY ||
1258                     level == G4X_WM_LEVEL_NORMAL)
1259                         continue;
1260
1261                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1262                                         raw->plane[plane_id]);
1263                 max_wm = g4x_fbc_fifo_size(level);
1264
1265                 /*
1266                  * FBC wm is not mandatory as we
1267                  * can always just disable its use.
1268                  */
1269                 if (wm > max_wm)
1270                         wm = USHRT_MAX;
1271
1272                 dirty |= raw->fbc != wm;
1273                 raw->fbc = wm;
1274         }
1275
1276         /* mark watermarks as invalid */
1277         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1278
1279         if (plane_id == PLANE_PRIMARY)
1280                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1281
1282  out:
1283         if (dirty) {
1284                 drm_dbg_kms(&dev_priv->drm,
1285                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1286                             plane->base.name,
1287                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1288                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1289                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1290
1291                 if (plane_id == PLANE_PRIMARY)
1292                         drm_dbg_kms(&dev_priv->drm,
1293                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1294                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1295                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1296         }
1297
1298         return dirty;
1299 }
1300
1301 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1302                                       enum plane_id plane_id, int level)
1303 {
1304         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1305
1306         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1307 }
1308
1309 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1310                                      int level)
1311 {
1312         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1313
1314         if (level > dev_priv->wm.max_level)
1315                 return false;
1316
1317         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1318                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1319                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1320 }
1321
1322 /* mark all levels starting from 'level' as invalid */
1323 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1324                                struct g4x_wm_state *wm_state, int level)
1325 {
1326         if (level <= G4X_WM_LEVEL_NORMAL) {
1327                 enum plane_id plane_id;
1328
1329                 for_each_plane_id_on_crtc(crtc, plane_id)
1330                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1331         }
1332
1333         if (level <= G4X_WM_LEVEL_SR) {
1334                 wm_state->cxsr = false;
1335                 wm_state->sr.cursor = USHRT_MAX;
1336                 wm_state->sr.plane = USHRT_MAX;
1337                 wm_state->sr.fbc = USHRT_MAX;
1338         }
1339
1340         if (level <= G4X_WM_LEVEL_HPLL) {
1341                 wm_state->hpll_en = false;
1342                 wm_state->hpll.cursor = USHRT_MAX;
1343                 wm_state->hpll.plane = USHRT_MAX;
1344                 wm_state->hpll.fbc = USHRT_MAX;
1345         }
1346 }
1347
1348 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1349 {
1350         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1351         struct intel_atomic_state *state =
1352                 to_intel_atomic_state(crtc_state->uapi.state);
1353         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1354         int num_active_planes = hweight8(crtc_state->active_planes &
1355                                          ~BIT(PLANE_CURSOR));
1356         const struct g4x_pipe_wm *raw;
1357         const struct intel_plane_state *old_plane_state;
1358         const struct intel_plane_state *new_plane_state;
1359         struct intel_plane *plane;
1360         enum plane_id plane_id;
1361         int i, level;
1362         unsigned int dirty = 0;
1363
1364         for_each_oldnew_intel_plane_in_state(state, plane,
1365                                              old_plane_state,
1366                                              new_plane_state, i) {
1367                 if (new_plane_state->hw.crtc != &crtc->base &&
1368                     old_plane_state->hw.crtc != &crtc->base)
1369                         continue;
1370
1371                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1372                         dirty |= BIT(plane->id);
1373         }
1374
1375         if (!dirty)
1376                 return 0;
1377
1378         level = G4X_WM_LEVEL_NORMAL;
1379         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1380                 goto out;
1381
1382         raw = &crtc_state->wm.g4x.raw[level];
1383         for_each_plane_id_on_crtc(crtc, plane_id)
1384                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1385
1386         level = G4X_WM_LEVEL_SR;
1387
1388         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1389                 goto out;
1390
1391         raw = &crtc_state->wm.g4x.raw[level];
1392         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1393         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1394         wm_state->sr.fbc = raw->fbc;
1395
1396         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1397
1398         level = G4X_WM_LEVEL_HPLL;
1399
1400         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1401                 goto out;
1402
1403         raw = &crtc_state->wm.g4x.raw[level];
1404         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1405         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1406         wm_state->hpll.fbc = raw->fbc;
1407
1408         wm_state->hpll_en = wm_state->cxsr;
1409
1410         level++;
1411
1412  out:
1413         if (level == G4X_WM_LEVEL_NORMAL)
1414                 return -EINVAL;
1415
1416         /* invalidate the higher levels */
1417         g4x_invalidate_wms(crtc, wm_state, level);
1418
1419         /*
1420          * Determine if the FBC watermark(s) can be used. IF
1421          * this isn't the case we prefer to disable the FBC
1422          ( watermark(s) rather than disable the SR/HPLL
1423          * level(s) entirely.
1424          */
1425         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1426
1427         if (level >= G4X_WM_LEVEL_SR &&
1428             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1429                 wm_state->fbc_en = false;
1430         else if (level >= G4X_WM_LEVEL_HPLL &&
1431                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1432                 wm_state->fbc_en = false;
1433
1434         return 0;
1435 }
1436
1437 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1438 {
1439         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1440         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1441         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1442         struct intel_atomic_state *intel_state =
1443                 to_intel_atomic_state(new_crtc_state->uapi.state);
1444         const struct intel_crtc_state *old_crtc_state =
1445                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1446         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1447         enum plane_id plane_id;
1448
1449         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1450                 *intermediate = *optimal;
1451
1452                 intermediate->cxsr = false;
1453                 intermediate->hpll_en = false;
1454                 goto out;
1455         }
1456
1457         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1458                 !new_crtc_state->disable_cxsr;
1459         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1460                 !new_crtc_state->disable_cxsr;
1461         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1462
1463         for_each_plane_id_on_crtc(crtc, plane_id) {
1464                 intermediate->wm.plane[plane_id] =
1465                         max(optimal->wm.plane[plane_id],
1466                             active->wm.plane[plane_id]);
1467
1468                 WARN_ON(intermediate->wm.plane[plane_id] >
1469                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1470         }
1471
1472         intermediate->sr.plane = max(optimal->sr.plane,
1473                                      active->sr.plane);
1474         intermediate->sr.cursor = max(optimal->sr.cursor,
1475                                       active->sr.cursor);
1476         intermediate->sr.fbc = max(optimal->sr.fbc,
1477                                    active->sr.fbc);
1478
1479         intermediate->hpll.plane = max(optimal->hpll.plane,
1480                                        active->hpll.plane);
1481         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1482                                         active->hpll.cursor);
1483         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1484                                      active->hpll.fbc);
1485
1486         WARN_ON((intermediate->sr.plane >
1487                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1488                  intermediate->sr.cursor >
1489                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1490                 intermediate->cxsr);
1491         WARN_ON((intermediate->sr.plane >
1492                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1493                  intermediate->sr.cursor >
1494                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1495                 intermediate->hpll_en);
1496
1497         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1498                 intermediate->fbc_en && intermediate->cxsr);
1499         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1500                 intermediate->fbc_en && intermediate->hpll_en);
1501
1502 out:
1503         /*
1504          * If our intermediate WM are identical to the final WM, then we can
1505          * omit the post-vblank programming; only update if it's different.
1506          */
1507         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1508                 new_crtc_state->wm.need_postvbl_update = true;
1509
1510         return 0;
1511 }
1512
1513 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1514                          struct g4x_wm_values *wm)
1515 {
1516         struct intel_crtc *crtc;
1517         int num_active_pipes = 0;
1518
1519         wm->cxsr = true;
1520         wm->hpll_en = true;
1521         wm->fbc_en = true;
1522
1523         for_each_intel_crtc(&dev_priv->drm, crtc) {
1524                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1525
1526                 if (!crtc->active)
1527                         continue;
1528
1529                 if (!wm_state->cxsr)
1530                         wm->cxsr = false;
1531                 if (!wm_state->hpll_en)
1532                         wm->hpll_en = false;
1533                 if (!wm_state->fbc_en)
1534                         wm->fbc_en = false;
1535
1536                 num_active_pipes++;
1537         }
1538
1539         if (num_active_pipes != 1) {
1540                 wm->cxsr = false;
1541                 wm->hpll_en = false;
1542                 wm->fbc_en = false;
1543         }
1544
1545         for_each_intel_crtc(&dev_priv->drm, crtc) {
1546                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1547                 enum pipe pipe = crtc->pipe;
1548
1549                 wm->pipe[pipe] = wm_state->wm;
1550                 if (crtc->active && wm->cxsr)
1551                         wm->sr = wm_state->sr;
1552                 if (crtc->active && wm->hpll_en)
1553                         wm->hpll = wm_state->hpll;
1554         }
1555 }
1556
1557 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1558 {
1559         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1560         struct g4x_wm_values new_wm = {};
1561
1562         g4x_merge_wm(dev_priv, &new_wm);
1563
1564         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1565                 return;
1566
1567         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1568                 _intel_set_memory_cxsr(dev_priv, false);
1569
1570         g4x_write_wm_values(dev_priv, &new_wm);
1571
1572         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1573                 _intel_set_memory_cxsr(dev_priv, true);
1574
1575         *old_wm = new_wm;
1576 }
1577
1578 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1579                                    struct intel_crtc *crtc)
1580 {
1581         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1582         const struct intel_crtc_state *crtc_state =
1583                 intel_atomic_get_new_crtc_state(state, crtc);
1584
1585         mutex_lock(&dev_priv->wm.wm_mutex);
1586         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1587         g4x_program_watermarks(dev_priv);
1588         mutex_unlock(&dev_priv->wm.wm_mutex);
1589 }
1590
1591 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1592                                     struct intel_crtc *crtc)
1593 {
1594         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1595         const struct intel_crtc_state *crtc_state =
1596                 intel_atomic_get_new_crtc_state(state, crtc);
1597
1598         if (!crtc_state->wm.need_postvbl_update)
1599                 return;
1600
1601         mutex_lock(&dev_priv->wm.wm_mutex);
1602         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1603         g4x_program_watermarks(dev_priv);
1604         mutex_unlock(&dev_priv->wm.wm_mutex);
1605 }
1606
1607 /* latency must be in 0.1us units. */
1608 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1609                                    unsigned int htotal,
1610                                    unsigned int width,
1611                                    unsigned int cpp,
1612                                    unsigned int latency)
1613 {
1614         unsigned int ret;
1615
1616         ret = intel_wm_method2(pixel_rate, htotal,
1617                                width, cpp, latency);
1618         ret = DIV_ROUND_UP(ret, 64);
1619
1620         return ret;
1621 }
1622
1623 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1624 {
1625         /* all latencies in usec */
1626         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1627
1628         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1629
1630         if (IS_CHERRYVIEW(dev_priv)) {
1631                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1632                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1633
1634                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1635         }
1636 }
1637
1638 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1639                                 const struct intel_plane_state *plane_state,
1640                                 int level)
1641 {
1642         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1643         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1644         const struct drm_display_mode *adjusted_mode =
1645                 &crtc_state->hw.adjusted_mode;
1646         unsigned int clock, htotal, cpp, width, wm;
1647
1648         if (dev_priv->wm.pri_latency[level] == 0)
1649                 return USHRT_MAX;
1650
1651         if (!intel_wm_plane_visible(crtc_state, plane_state))
1652                 return 0;
1653
1654         cpp = plane_state->hw.fb->format->cpp[0];
1655         clock = adjusted_mode->crtc_clock;
1656         htotal = adjusted_mode->crtc_htotal;
1657         width = crtc_state->pipe_src_w;
1658
1659         if (plane->id == PLANE_CURSOR) {
1660                 /*
1661                  * FIXME the formula gives values that are
1662                  * too big for the cursor FIFO, and hence we
1663                  * would never be able to use cursors. For
1664                  * now just hardcode the watermark.
1665                  */
1666                 wm = 63;
1667         } else {
1668                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1669                                     dev_priv->wm.pri_latency[level] * 10);
1670         }
1671
1672         return min_t(unsigned int, wm, USHRT_MAX);
1673 }
1674
1675 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1676 {
1677         return (active_planes & (BIT(PLANE_SPRITE0) |
1678                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1679 }
1680
1681 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1682 {
1683         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1684         const struct g4x_pipe_wm *raw =
1685                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1686         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1687         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1688         int num_active_planes = hweight8(active_planes);
1689         const int fifo_size = 511;
1690         int fifo_extra, fifo_left = fifo_size;
1691         int sprite0_fifo_extra = 0;
1692         unsigned int total_rate;
1693         enum plane_id plane_id;
1694
1695         /*
1696          * When enabling sprite0 after sprite1 has already been enabled
1697          * we tend to get an underrun unless sprite0 already has some
1698          * FIFO space allcoated. Hence we always allocate at least one
1699          * cacheline for sprite0 whenever sprite1 is enabled.
1700          *
1701          * All other plane enable sequences appear immune to this problem.
1702          */
1703         if (vlv_need_sprite0_fifo_workaround(active_planes))
1704                 sprite0_fifo_extra = 1;
1705
1706         total_rate = raw->plane[PLANE_PRIMARY] +
1707                 raw->plane[PLANE_SPRITE0] +
1708                 raw->plane[PLANE_SPRITE1] +
1709                 sprite0_fifo_extra;
1710
1711         if (total_rate > fifo_size)
1712                 return -EINVAL;
1713
1714         if (total_rate == 0)
1715                 total_rate = 1;
1716
1717         for_each_plane_id_on_crtc(crtc, plane_id) {
1718                 unsigned int rate;
1719
1720                 if ((active_planes & BIT(plane_id)) == 0) {
1721                         fifo_state->plane[plane_id] = 0;
1722                         continue;
1723                 }
1724
1725                 rate = raw->plane[plane_id];
1726                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1727                 fifo_left -= fifo_state->plane[plane_id];
1728         }
1729
1730         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1731         fifo_left -= sprite0_fifo_extra;
1732
1733         fifo_state->plane[PLANE_CURSOR] = 63;
1734
1735         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1736
1737         /* spread the remainder evenly */
1738         for_each_plane_id_on_crtc(crtc, plane_id) {
1739                 int plane_extra;
1740
1741                 if (fifo_left == 0)
1742                         break;
1743
1744                 if ((active_planes & BIT(plane_id)) == 0)
1745                         continue;
1746
1747                 plane_extra = min(fifo_extra, fifo_left);
1748                 fifo_state->plane[plane_id] += plane_extra;
1749                 fifo_left -= plane_extra;
1750         }
1751
1752         WARN_ON(active_planes != 0 && fifo_left != 0);
1753
1754         /* give it all to the first plane if none are active */
1755         if (active_planes == 0) {
1756                 WARN_ON(fifo_left != fifo_size);
1757                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1758         }
1759
1760         return 0;
1761 }
1762
1763 /* mark all levels starting from 'level' as invalid */
1764 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1765                                struct vlv_wm_state *wm_state, int level)
1766 {
1767         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1768
1769         for (; level < intel_wm_num_levels(dev_priv); level++) {
1770                 enum plane_id plane_id;
1771
1772                 for_each_plane_id_on_crtc(crtc, plane_id)
1773                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1774
1775                 wm_state->sr[level].cursor = USHRT_MAX;
1776                 wm_state->sr[level].plane = USHRT_MAX;
1777         }
1778 }
1779
1780 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1781 {
1782         if (wm > fifo_size)
1783                 return USHRT_MAX;
1784         else
1785                 return fifo_size - wm;
1786 }
1787
1788 /*
1789  * Starting from 'level' set all higher
1790  * levels to 'value' in the "raw" watermarks.
1791  */
1792 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1793                                  int level, enum plane_id plane_id, u16 value)
1794 {
1795         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1796         int num_levels = intel_wm_num_levels(dev_priv);
1797         bool dirty = false;
1798
1799         for (; level < num_levels; level++) {
1800                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1801
1802                 dirty |= raw->plane[plane_id] != value;
1803                 raw->plane[plane_id] = value;
1804         }
1805
1806         return dirty;
1807 }
1808
1809 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1810                                      const struct intel_plane_state *plane_state)
1811 {
1812         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1813         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1814         enum plane_id plane_id = plane->id;
1815         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1816         int level;
1817         bool dirty = false;
1818
1819         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1820                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1821                 goto out;
1822         }
1823
1824         for (level = 0; level < num_levels; level++) {
1825                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1826                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1827                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1828
1829                 if (wm > max_wm)
1830                         break;
1831
1832                 dirty |= raw->plane[plane_id] != wm;
1833                 raw->plane[plane_id] = wm;
1834         }
1835
1836         /* mark all higher levels as invalid */
1837         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1838
1839 out:
1840         if (dirty)
1841                 drm_dbg_kms(&dev_priv->drm,
1842                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1843                             plane->base.name,
1844                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1845                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1846                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1847
1848         return dirty;
1849 }
1850
1851 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1852                                       enum plane_id plane_id, int level)
1853 {
1854         const struct g4x_pipe_wm *raw =
1855                 &crtc_state->wm.vlv.raw[level];
1856         const struct vlv_fifo_state *fifo_state =
1857                 &crtc_state->wm.vlv.fifo_state;
1858
1859         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1860 }
1861
1862 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1863 {
1864         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1865                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1866                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1867                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1868 }
1869
1870 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1871 {
1872         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1873         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1874         struct intel_atomic_state *state =
1875                 to_intel_atomic_state(crtc_state->uapi.state);
1876         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1877         const struct vlv_fifo_state *fifo_state =
1878                 &crtc_state->wm.vlv.fifo_state;
1879         int num_active_planes = hweight8(crtc_state->active_planes &
1880                                          ~BIT(PLANE_CURSOR));
1881         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1882         const struct intel_plane_state *old_plane_state;
1883         const struct intel_plane_state *new_plane_state;
1884         struct intel_plane *plane;
1885         enum plane_id plane_id;
1886         int level, ret, i;
1887         unsigned int dirty = 0;
1888
1889         for_each_oldnew_intel_plane_in_state(state, plane,
1890                                              old_plane_state,
1891                                              new_plane_state, i) {
1892                 if (new_plane_state->hw.crtc != &crtc->base &&
1893                     old_plane_state->hw.crtc != &crtc->base)
1894                         continue;
1895
1896                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1897                         dirty |= BIT(plane->id);
1898         }
1899
1900         /*
1901          * DSPARB registers may have been reset due to the
1902          * power well being turned off. Make sure we restore
1903          * them to a consistent state even if no primary/sprite
1904          * planes are initially active.
1905          */
1906         if (needs_modeset)
1907                 crtc_state->fifo_changed = true;
1908
1909         if (!dirty)
1910                 return 0;
1911
1912         /* cursor changes don't warrant a FIFO recompute */
1913         if (dirty & ~BIT(PLANE_CURSOR)) {
1914                 const struct intel_crtc_state *old_crtc_state =
1915                         intel_atomic_get_old_crtc_state(state, crtc);
1916                 const struct vlv_fifo_state *old_fifo_state =
1917                         &old_crtc_state->wm.vlv.fifo_state;
1918
1919                 ret = vlv_compute_fifo(crtc_state);
1920                 if (ret)
1921                         return ret;
1922
1923                 if (needs_modeset ||
1924                     memcmp(old_fifo_state, fifo_state,
1925                            sizeof(*fifo_state)) != 0)
1926                         crtc_state->fifo_changed = true;
1927         }
1928
1929         /* initially allow all levels */
1930         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1931         /*
1932          * Note that enabling cxsr with no primary/sprite planes
1933          * enabled can wedge the pipe. Hence we only allow cxsr
1934          * with exactly one enabled primary/sprite plane.
1935          */
1936         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1937
1938         for (level = 0; level < wm_state->num_levels; level++) {
1939                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1940                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1941
1942                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1943                         break;
1944
1945                 for_each_plane_id_on_crtc(crtc, plane_id) {
1946                         wm_state->wm[level].plane[plane_id] =
1947                                 vlv_invert_wm_value(raw->plane[plane_id],
1948                                                     fifo_state->plane[plane_id]);
1949                 }
1950
1951                 wm_state->sr[level].plane =
1952                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1953                                                  raw->plane[PLANE_SPRITE0],
1954                                                  raw->plane[PLANE_SPRITE1]),
1955                                             sr_fifo_size);
1956
1957                 wm_state->sr[level].cursor =
1958                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1959                                             63);
1960         }
1961
1962         if (level == 0)
1963                 return -EINVAL;
1964
1965         /* limit to only levels we can actually handle */
1966         wm_state->num_levels = level;
1967
1968         /* invalidate the higher levels */
1969         vlv_invalidate_wms(crtc, wm_state, level);
1970
1971         return 0;
1972 }
1973
1974 #define VLV_FIFO(plane, value) \
1975         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1976
1977 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1978                                    struct intel_crtc *crtc)
1979 {
1980         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1981         struct intel_uncore *uncore = &dev_priv->uncore;
1982         const struct intel_crtc_state *crtc_state =
1983                 intel_atomic_get_new_crtc_state(state, crtc);
1984         const struct vlv_fifo_state *fifo_state =
1985                 &crtc_state->wm.vlv.fifo_state;
1986         int sprite0_start, sprite1_start, fifo_size;
1987         u32 dsparb, dsparb2, dsparb3;
1988
1989         if (!crtc_state->fifo_changed)
1990                 return;
1991
1992         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1993         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1994         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1995
1996         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1997         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1998
1999         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2000
2001         /*
2002          * uncore.lock serves a double purpose here. It allows us to
2003          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2004          * it protects the DSPARB registers from getting clobbered by
2005          * parallel updates from multiple pipes.
2006          *
2007          * intel_pipe_update_start() has already disabled interrupts
2008          * for us, so a plain spin_lock() is sufficient here.
2009          */
2010         spin_lock(&uncore->lock);
2011
2012         switch (crtc->pipe) {
2013         case PIPE_A:
2014                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2015                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2016
2017                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2018                             VLV_FIFO(SPRITEB, 0xff));
2019                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2020                            VLV_FIFO(SPRITEB, sprite1_start));
2021
2022                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2023                              VLV_FIFO(SPRITEB_HI, 0x1));
2024                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2025                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2026
2027                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2028                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2029                 break;
2030         case PIPE_B:
2031                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2032                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2033
2034                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2035                             VLV_FIFO(SPRITED, 0xff));
2036                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2037                            VLV_FIFO(SPRITED, sprite1_start));
2038
2039                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2040                              VLV_FIFO(SPRITED_HI, 0xff));
2041                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2042                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2043
2044                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2045                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2046                 break;
2047         case PIPE_C:
2048                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2049                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2050
2051                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2052                              VLV_FIFO(SPRITEF, 0xff));
2053                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2054                             VLV_FIFO(SPRITEF, sprite1_start));
2055
2056                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2057                              VLV_FIFO(SPRITEF_HI, 0xff));
2058                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2059                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2060
2061                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2062                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2063                 break;
2064         default:
2065                 break;
2066         }
2067
2068         intel_uncore_posting_read_fw(uncore, DSPARB);
2069
2070         spin_unlock(&uncore->lock);
2071 }
2072
2073 #undef VLV_FIFO
2074
2075 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2076 {
2077         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2078         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2079         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2080         struct intel_atomic_state *intel_state =
2081                 to_intel_atomic_state(new_crtc_state->uapi.state);
2082         const struct intel_crtc_state *old_crtc_state =
2083                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2084         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2085         int level;
2086
2087         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2088                 *intermediate = *optimal;
2089
2090                 intermediate->cxsr = false;
2091                 goto out;
2092         }
2093
2094         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2095         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2096                 !new_crtc_state->disable_cxsr;
2097
2098         for (level = 0; level < intermediate->num_levels; level++) {
2099                 enum plane_id plane_id;
2100
2101                 for_each_plane_id_on_crtc(crtc, plane_id) {
2102                         intermediate->wm[level].plane[plane_id] =
2103                                 min(optimal->wm[level].plane[plane_id],
2104                                     active->wm[level].plane[plane_id]);
2105                 }
2106
2107                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2108                                                     active->sr[level].plane);
2109                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2110                                                      active->sr[level].cursor);
2111         }
2112
2113         vlv_invalidate_wms(crtc, intermediate, level);
2114
2115 out:
2116         /*
2117          * If our intermediate WM are identical to the final WM, then we can
2118          * omit the post-vblank programming; only update if it's different.
2119          */
2120         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2121                 new_crtc_state->wm.need_postvbl_update = true;
2122
2123         return 0;
2124 }
2125
2126 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2127                          struct vlv_wm_values *wm)
2128 {
2129         struct intel_crtc *crtc;
2130         int num_active_pipes = 0;
2131
2132         wm->level = dev_priv->wm.max_level;
2133         wm->cxsr = true;
2134
2135         for_each_intel_crtc(&dev_priv->drm, crtc) {
2136                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2137
2138                 if (!crtc->active)
2139                         continue;
2140
2141                 if (!wm_state->cxsr)
2142                         wm->cxsr = false;
2143
2144                 num_active_pipes++;
2145                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2146         }
2147
2148         if (num_active_pipes != 1)
2149                 wm->cxsr = false;
2150
2151         if (num_active_pipes > 1)
2152                 wm->level = VLV_WM_LEVEL_PM2;
2153
2154         for_each_intel_crtc(&dev_priv->drm, crtc) {
2155                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2156                 enum pipe pipe = crtc->pipe;
2157
2158                 wm->pipe[pipe] = wm_state->wm[wm->level];
2159                 if (crtc->active && wm->cxsr)
2160                         wm->sr = wm_state->sr[wm->level];
2161
2162                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2163                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2164                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2165                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2166         }
2167 }
2168
2169 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2170 {
2171         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2172         struct vlv_wm_values new_wm = {};
2173
2174         vlv_merge_wm(dev_priv, &new_wm);
2175
2176         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2177                 return;
2178
2179         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2180                 chv_set_memory_dvfs(dev_priv, false);
2181
2182         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2183                 chv_set_memory_pm5(dev_priv, false);
2184
2185         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2186                 _intel_set_memory_cxsr(dev_priv, false);
2187
2188         vlv_write_wm_values(dev_priv, &new_wm);
2189
2190         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2191                 _intel_set_memory_cxsr(dev_priv, true);
2192
2193         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2194                 chv_set_memory_pm5(dev_priv, true);
2195
2196         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2197                 chv_set_memory_dvfs(dev_priv, true);
2198
2199         *old_wm = new_wm;
2200 }
2201
2202 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2203                                    struct intel_crtc *crtc)
2204 {
2205         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2206         const struct intel_crtc_state *crtc_state =
2207                 intel_atomic_get_new_crtc_state(state, crtc);
2208
2209         mutex_lock(&dev_priv->wm.wm_mutex);
2210         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2211         vlv_program_watermarks(dev_priv);
2212         mutex_unlock(&dev_priv->wm.wm_mutex);
2213 }
2214
2215 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2216                                     struct intel_crtc *crtc)
2217 {
2218         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2219         const struct intel_crtc_state *crtc_state =
2220                 intel_atomic_get_new_crtc_state(state, crtc);
2221
2222         if (!crtc_state->wm.need_postvbl_update)
2223                 return;
2224
2225         mutex_lock(&dev_priv->wm.wm_mutex);
2226         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2227         vlv_program_watermarks(dev_priv);
2228         mutex_unlock(&dev_priv->wm.wm_mutex);
2229 }
2230
2231 static void i965_update_wm(struct intel_crtc *unused_crtc)
2232 {
2233         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2234         struct intel_crtc *crtc;
2235         int srwm = 1;
2236         int cursor_sr = 16;
2237         bool cxsr_enabled;
2238
2239         /* Calc sr entries for one plane configs */
2240         crtc = single_enabled_crtc(dev_priv);
2241         if (crtc) {
2242                 /* self-refresh has much higher latency */
2243                 static const int sr_latency_ns = 12000;
2244                 const struct drm_display_mode *adjusted_mode =
2245                         &crtc->config->hw.adjusted_mode;
2246                 const struct drm_framebuffer *fb =
2247                         crtc->base.primary->state->fb;
2248                 int clock = adjusted_mode->crtc_clock;
2249                 int htotal = adjusted_mode->crtc_htotal;
2250                 int hdisplay = crtc->config->pipe_src_w;
2251                 int cpp = fb->format->cpp[0];
2252                 int entries;
2253
2254                 entries = intel_wm_method2(clock, htotal,
2255                                            hdisplay, cpp, sr_latency_ns / 100);
2256                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2257                 srwm = I965_FIFO_SIZE - entries;
2258                 if (srwm < 0)
2259                         srwm = 1;
2260                 srwm &= 0x1ff;
2261                 drm_dbg_kms(&dev_priv->drm,
2262                             "self-refresh entries: %d, wm: %d\n",
2263                             entries, srwm);
2264
2265                 entries = intel_wm_method2(clock, htotal,
2266                                            crtc->base.cursor->state->crtc_w, 4,
2267                                            sr_latency_ns / 100);
2268                 entries = DIV_ROUND_UP(entries,
2269                                        i965_cursor_wm_info.cacheline_size) +
2270                         i965_cursor_wm_info.guard_size;
2271
2272                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2273                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2274                         cursor_sr = i965_cursor_wm_info.max_wm;
2275
2276                 drm_dbg_kms(&dev_priv->drm,
2277                             "self-refresh watermark: display plane %d "
2278                             "cursor %d\n", srwm, cursor_sr);
2279
2280                 cxsr_enabled = true;
2281         } else {
2282                 cxsr_enabled = false;
2283                 /* Turn off self refresh if both pipes are enabled */
2284                 intel_set_memory_cxsr(dev_priv, false);
2285         }
2286
2287         drm_dbg_kms(&dev_priv->drm,
2288                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2289                     srwm);
2290
2291         /* 965 has limitations... */
2292         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2293                    FW_WM(8, CURSORB) |
2294                    FW_WM(8, PLANEB) |
2295                    FW_WM(8, PLANEA));
2296         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2297                    FW_WM(8, PLANEC_OLD));
2298         /* update cursor SR watermark */
2299         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2300
2301         if (cxsr_enabled)
2302                 intel_set_memory_cxsr(dev_priv, true);
2303 }
2304
2305 #undef FW_WM
2306
2307 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2308 {
2309         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2310         const struct intel_watermark_params *wm_info;
2311         u32 fwater_lo;
2312         u32 fwater_hi;
2313         int cwm, srwm = 1;
2314         int fifo_size;
2315         int planea_wm, planeb_wm;
2316         struct intel_crtc *crtc, *enabled = NULL;
2317
2318         if (IS_I945GM(dev_priv))
2319                 wm_info = &i945_wm_info;
2320         else if (!IS_GEN(dev_priv, 2))
2321                 wm_info = &i915_wm_info;
2322         else
2323                 wm_info = &i830_a_wm_info;
2324
2325         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2326         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2327         if (intel_crtc_active(crtc)) {
2328                 const struct drm_display_mode *adjusted_mode =
2329                         &crtc->config->hw.adjusted_mode;
2330                 const struct drm_framebuffer *fb =
2331                         crtc->base.primary->state->fb;
2332                 int cpp;
2333
2334                 if (IS_GEN(dev_priv, 2))
2335                         cpp = 4;
2336                 else
2337                         cpp = fb->format->cpp[0];
2338
2339                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2340                                                wm_info, fifo_size, cpp,
2341                                                pessimal_latency_ns);
2342                 enabled = crtc;
2343         } else {
2344                 planea_wm = fifo_size - wm_info->guard_size;
2345                 if (planea_wm > (long)wm_info->max_wm)
2346                         planea_wm = wm_info->max_wm;
2347         }
2348
2349         if (IS_GEN(dev_priv, 2))
2350                 wm_info = &i830_bc_wm_info;
2351
2352         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2353         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2354         if (intel_crtc_active(crtc)) {
2355                 const struct drm_display_mode *adjusted_mode =
2356                         &crtc->config->hw.adjusted_mode;
2357                 const struct drm_framebuffer *fb =
2358                         crtc->base.primary->state->fb;
2359                 int cpp;
2360
2361                 if (IS_GEN(dev_priv, 2))
2362                         cpp = 4;
2363                 else
2364                         cpp = fb->format->cpp[0];
2365
2366                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2367                                                wm_info, fifo_size, cpp,
2368                                                pessimal_latency_ns);
2369                 if (enabled == NULL)
2370                         enabled = crtc;
2371                 else
2372                         enabled = NULL;
2373         } else {
2374                 planeb_wm = fifo_size - wm_info->guard_size;
2375                 if (planeb_wm > (long)wm_info->max_wm)
2376                         planeb_wm = wm_info->max_wm;
2377         }
2378
2379         drm_dbg_kms(&dev_priv->drm,
2380                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2381
2382         if (IS_I915GM(dev_priv) && enabled) {
2383                 struct drm_i915_gem_object *obj;
2384
2385                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2386
2387                 /* self-refresh seems busted with untiled */
2388                 if (!i915_gem_object_is_tiled(obj))
2389                         enabled = NULL;
2390         }
2391
2392         /*
2393          * Overlay gets an aggressive default since video jitter is bad.
2394          */
2395         cwm = 2;
2396
2397         /* Play safe and disable self-refresh before adjusting watermarks. */
2398         intel_set_memory_cxsr(dev_priv, false);
2399
2400         /* Calc sr entries for one plane configs */
2401         if (HAS_FW_BLC(dev_priv) && enabled) {
2402                 /* self-refresh has much higher latency */
2403                 static const int sr_latency_ns = 6000;
2404                 const struct drm_display_mode *adjusted_mode =
2405                         &enabled->config->hw.adjusted_mode;
2406                 const struct drm_framebuffer *fb =
2407                         enabled->base.primary->state->fb;
2408                 int clock = adjusted_mode->crtc_clock;
2409                 int htotal = adjusted_mode->crtc_htotal;
2410                 int hdisplay = enabled->config->pipe_src_w;
2411                 int cpp;
2412                 int entries;
2413
2414                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2415                         cpp = 4;
2416                 else
2417                         cpp = fb->format->cpp[0];
2418
2419                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2420                                            sr_latency_ns / 100);
2421                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2422                 drm_dbg_kms(&dev_priv->drm,
2423                             "self-refresh entries: %d\n", entries);
2424                 srwm = wm_info->fifo_size - entries;
2425                 if (srwm < 0)
2426                         srwm = 1;
2427
2428                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2429                         I915_WRITE(FW_BLC_SELF,
2430                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2431                 else
2432                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2433         }
2434
2435         drm_dbg_kms(&dev_priv->drm,
2436                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2437                      planea_wm, planeb_wm, cwm, srwm);
2438
2439         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2440         fwater_hi = (cwm & 0x1f);
2441
2442         /* Set request length to 8 cachelines per fetch */
2443         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2444         fwater_hi = fwater_hi | (1 << 8);
2445
2446         I915_WRITE(FW_BLC, fwater_lo);
2447         I915_WRITE(FW_BLC2, fwater_hi);
2448
2449         if (enabled)
2450                 intel_set_memory_cxsr(dev_priv, true);
2451 }
2452
2453 static void i845_update_wm(struct intel_crtc *unused_crtc)
2454 {
2455         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2456         struct intel_crtc *crtc;
2457         const struct drm_display_mode *adjusted_mode;
2458         u32 fwater_lo;
2459         int planea_wm;
2460
2461         crtc = single_enabled_crtc(dev_priv);
2462         if (crtc == NULL)
2463                 return;
2464
2465         adjusted_mode = &crtc->config->hw.adjusted_mode;
2466         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2467                                        &i845_wm_info,
2468                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2469                                        4, pessimal_latency_ns);
2470         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2471         fwater_lo |= (3<<8) | planea_wm;
2472
2473         drm_dbg_kms(&dev_priv->drm,
2474                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2475
2476         I915_WRITE(FW_BLC, fwater_lo);
2477 }
2478
2479 /* latency must be in 0.1us units. */
2480 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2481                                    unsigned int cpp,
2482                                    unsigned int latency)
2483 {
2484         unsigned int ret;
2485
2486         ret = intel_wm_method1(pixel_rate, cpp, latency);
2487         ret = DIV_ROUND_UP(ret, 64) + 2;
2488
2489         return ret;
2490 }
2491
2492 /* latency must be in 0.1us units. */
2493 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2494                                    unsigned int htotal,
2495                                    unsigned int width,
2496                                    unsigned int cpp,
2497                                    unsigned int latency)
2498 {
2499         unsigned int ret;
2500
2501         ret = intel_wm_method2(pixel_rate, htotal,
2502                                width, cpp, latency);
2503         ret = DIV_ROUND_UP(ret, 64) + 2;
2504
2505         return ret;
2506 }
2507
2508 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2509 {
2510         /*
2511          * Neither of these should be possible since this function shouldn't be
2512          * called if the CRTC is off or the plane is invisible.  But let's be
2513          * extra paranoid to avoid a potential divide-by-zero if we screw up
2514          * elsewhere in the driver.
2515          */
2516         if (WARN_ON(!cpp))
2517                 return 0;
2518         if (WARN_ON(!horiz_pixels))
2519                 return 0;
2520
2521         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2522 }
2523
2524 struct ilk_wm_maximums {
2525         u16 pri;
2526         u16 spr;
2527         u16 cur;
2528         u16 fbc;
2529 };
2530
2531 /*
2532  * For both WM_PIPE and WM_LP.
2533  * mem_value must be in 0.1us units.
2534  */
2535 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2536                               const struct intel_plane_state *plane_state,
2537                               u32 mem_value, bool is_lp)
2538 {
2539         u32 method1, method2;
2540         int cpp;
2541
2542         if (mem_value == 0)
2543                 return U32_MAX;
2544
2545         if (!intel_wm_plane_visible(crtc_state, plane_state))
2546                 return 0;
2547
2548         cpp = plane_state->hw.fb->format->cpp[0];
2549
2550         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2551
2552         if (!is_lp)
2553                 return method1;
2554
2555         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2556                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2557                                  drm_rect_width(&plane_state->uapi.dst),
2558                                  cpp, mem_value);
2559
2560         return min(method1, method2);
2561 }
2562
2563 /*
2564  * For both WM_PIPE and WM_LP.
2565  * mem_value must be in 0.1us units.
2566  */
2567 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2568                               const struct intel_plane_state *plane_state,
2569                               u32 mem_value)
2570 {
2571         u32 method1, method2;
2572         int cpp;
2573
2574         if (mem_value == 0)
2575                 return U32_MAX;
2576
2577         if (!intel_wm_plane_visible(crtc_state, plane_state))
2578                 return 0;
2579
2580         cpp = plane_state->hw.fb->format->cpp[0];
2581
2582         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2583         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2584                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2585                                  drm_rect_width(&plane_state->uapi.dst),
2586                                  cpp, mem_value);
2587         return min(method1, method2);
2588 }
2589
2590 /*
2591  * For both WM_PIPE and WM_LP.
2592  * mem_value must be in 0.1us units.
2593  */
2594 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2595                               const struct intel_plane_state *plane_state,
2596                               u32 mem_value)
2597 {
2598         int cpp;
2599
2600         if (mem_value == 0)
2601                 return U32_MAX;
2602
2603         if (!intel_wm_plane_visible(crtc_state, plane_state))
2604                 return 0;
2605
2606         cpp = plane_state->hw.fb->format->cpp[0];
2607
2608         return ilk_wm_method2(crtc_state->pixel_rate,
2609                               crtc_state->hw.adjusted_mode.crtc_htotal,
2610                               drm_rect_width(&plane_state->uapi.dst),
2611                               cpp, mem_value);
2612 }
2613
2614 /* Only for WM_LP. */
2615 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2616                               const struct intel_plane_state *plane_state,
2617                               u32 pri_val)
2618 {
2619         int cpp;
2620
2621         if (!intel_wm_plane_visible(crtc_state, plane_state))
2622                 return 0;
2623
2624         cpp = plane_state->hw.fb->format->cpp[0];
2625
2626         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2627                           cpp);
2628 }
2629
2630 static unsigned int
2631 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2632 {
2633         if (INTEL_GEN(dev_priv) >= 8)
2634                 return 3072;
2635         else if (INTEL_GEN(dev_priv) >= 7)
2636                 return 768;
2637         else
2638                 return 512;
2639 }
2640
2641 static unsigned int
2642 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2643                      int level, bool is_sprite)
2644 {
2645         if (INTEL_GEN(dev_priv) >= 8)
2646                 /* BDW primary/sprite plane watermarks */
2647                 return level == 0 ? 255 : 2047;
2648         else if (INTEL_GEN(dev_priv) >= 7)
2649                 /* IVB/HSW primary/sprite plane watermarks */
2650                 return level == 0 ? 127 : 1023;
2651         else if (!is_sprite)
2652                 /* ILK/SNB primary plane watermarks */
2653                 return level == 0 ? 127 : 511;
2654         else
2655                 /* ILK/SNB sprite plane watermarks */
2656                 return level == 0 ? 63 : 255;
2657 }
2658
2659 static unsigned int
2660 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2661 {
2662         if (INTEL_GEN(dev_priv) >= 7)
2663                 return level == 0 ? 63 : 255;
2664         else
2665                 return level == 0 ? 31 : 63;
2666 }
2667
2668 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2669 {
2670         if (INTEL_GEN(dev_priv) >= 8)
2671                 return 31;
2672         else
2673                 return 15;
2674 }
2675
2676 /* Calculate the maximum primary/sprite plane watermark */
2677 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2678                                      int level,
2679                                      const struct intel_wm_config *config,
2680                                      enum intel_ddb_partitioning ddb_partitioning,
2681                                      bool is_sprite)
2682 {
2683         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2684
2685         /* if sprites aren't enabled, sprites get nothing */
2686         if (is_sprite && !config->sprites_enabled)
2687                 return 0;
2688
2689         /* HSW allows LP1+ watermarks even with multiple pipes */
2690         if (level == 0 || config->num_pipes_active > 1) {
2691                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2692
2693                 /*
2694                  * For some reason the non self refresh
2695                  * FIFO size is only half of the self
2696                  * refresh FIFO size on ILK/SNB.
2697                  */
2698                 if (INTEL_GEN(dev_priv) <= 6)
2699                         fifo_size /= 2;
2700         }
2701
2702         if (config->sprites_enabled) {
2703                 /* level 0 is always calculated with 1:1 split */
2704                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2705                         if (is_sprite)
2706                                 fifo_size *= 5;
2707                         fifo_size /= 6;
2708                 } else {
2709                         fifo_size /= 2;
2710                 }
2711         }
2712
2713         /* clamp to max that the registers can hold */
2714         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2715 }
2716
2717 /* Calculate the maximum cursor plane watermark */
2718 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2719                                       int level,
2720                                       const struct intel_wm_config *config)
2721 {
2722         /* HSW LP1+ watermarks w/ multiple pipes */
2723         if (level > 0 && config->num_pipes_active > 1)
2724                 return 64;
2725
2726         /* otherwise just report max that registers can hold */
2727         return ilk_cursor_wm_reg_max(dev_priv, level);
2728 }
2729
2730 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2731                                     int level,
2732                                     const struct intel_wm_config *config,
2733                                     enum intel_ddb_partitioning ddb_partitioning,
2734                                     struct ilk_wm_maximums *max)
2735 {
2736         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2737         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2738         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2739         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2740 }
2741
2742 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2743                                         int level,
2744                                         struct ilk_wm_maximums *max)
2745 {
2746         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2747         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2748         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2749         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2750 }
2751
2752 static bool ilk_validate_wm_level(int level,
2753                                   const struct ilk_wm_maximums *max,
2754                                   struct intel_wm_level *result)
2755 {
2756         bool ret;
2757
2758         /* already determined to be invalid? */
2759         if (!result->enable)
2760                 return false;
2761
2762         result->enable = result->pri_val <= max->pri &&
2763                          result->spr_val <= max->spr &&
2764                          result->cur_val <= max->cur;
2765
2766         ret = result->enable;
2767
2768         /*
2769          * HACK until we can pre-compute everything,
2770          * and thus fail gracefully if LP0 watermarks
2771          * are exceeded...
2772          */
2773         if (level == 0 && !result->enable) {
2774                 if (result->pri_val > max->pri)
2775                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2776                                       level, result->pri_val, max->pri);
2777                 if (result->spr_val > max->spr)
2778                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2779                                       level, result->spr_val, max->spr);
2780                 if (result->cur_val > max->cur)
2781                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2782                                       level, result->cur_val, max->cur);
2783
2784                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2785                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2786                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2787                 result->enable = true;
2788         }
2789
2790         return ret;
2791 }
2792
2793 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2794                                  const struct intel_crtc *crtc,
2795                                  int level,
2796                                  struct intel_crtc_state *crtc_state,
2797                                  const struct intel_plane_state *pristate,
2798                                  const struct intel_plane_state *sprstate,
2799                                  const struct intel_plane_state *curstate,
2800                                  struct intel_wm_level *result)
2801 {
2802         u16 pri_latency = dev_priv->wm.pri_latency[level];
2803         u16 spr_latency = dev_priv->wm.spr_latency[level];
2804         u16 cur_latency = dev_priv->wm.cur_latency[level];
2805
2806         /* WM1+ latency values stored in 0.5us units */
2807         if (level > 0) {
2808                 pri_latency *= 5;
2809                 spr_latency *= 5;
2810                 cur_latency *= 5;
2811         }
2812
2813         if (pristate) {
2814                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2815                                                      pri_latency, level);
2816                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2817         }
2818
2819         if (sprstate)
2820                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2821
2822         if (curstate)
2823                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2824
2825         result->enable = true;
2826 }
2827
2828 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2829                                   u16 wm[8])
2830 {
2831         struct intel_uncore *uncore = &dev_priv->uncore;
2832
2833         if (INTEL_GEN(dev_priv) >= 9) {
2834                 u32 val;
2835                 int ret, i;
2836                 int level, max_level = ilk_wm_max_level(dev_priv);
2837
2838                 /* read the first set of memory latencies[0:3] */
2839                 val = 0; /* data0 to be programmed to 0 for first set */
2840                 ret = sandybridge_pcode_read(dev_priv,
2841                                              GEN9_PCODE_READ_MEM_LATENCY,
2842                                              &val, NULL);
2843
2844                 if (ret) {
2845                         drm_err(&dev_priv->drm,
2846                                 "SKL Mailbox read error = %d\n", ret);
2847                         return;
2848                 }
2849
2850                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2851                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2852                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2853                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2854                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2855                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2856                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2857
2858                 /* read the second set of memory latencies[4:7] */
2859                 val = 1; /* data0 to be programmed to 1 for second set */
2860                 ret = sandybridge_pcode_read(dev_priv,
2861                                              GEN9_PCODE_READ_MEM_LATENCY,
2862                                              &val, NULL);
2863                 if (ret) {
2864                         drm_err(&dev_priv->drm,
2865                                 "SKL Mailbox read error = %d\n", ret);
2866                         return;
2867                 }
2868
2869                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2870                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2871                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2872                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2873                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2874                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2875                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2876
2877                 /*
2878                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2879                  * need to be disabled. We make sure to sanitize the values out
2880                  * of the punit to satisfy this requirement.
2881                  */
2882                 for (level = 1; level <= max_level; level++) {
2883                         if (wm[level] == 0) {
2884                                 for (i = level + 1; i <= max_level; i++)
2885                                         wm[i] = 0;
2886                                 break;
2887                         }
2888                 }
2889
2890                 /*
2891                  * WaWmMemoryReadLatency:skl+,glk
2892                  *
2893                  * punit doesn't take into account the read latency so we need
2894                  * to add 2us to the various latency levels we retrieve from the
2895                  * punit when level 0 response data us 0us.
2896                  */
2897                 if (wm[0] == 0) {
2898                         wm[0] += 2;
2899                         for (level = 1; level <= max_level; level++) {
2900                                 if (wm[level] == 0)
2901                                         break;
2902                                 wm[level] += 2;
2903                         }
2904                 }
2905
2906                 /*
2907                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2908                  * If we could not get dimm info enable this WA to prevent from
2909                  * any underrun. If not able to get Dimm info assume 16GB dimm
2910                  * to avoid any underrun.
2911                  */
2912                 if (dev_priv->dram_info.is_16gb_dimm)
2913                         wm[0] += 1;
2914
2915         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2916                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2917
2918                 wm[0] = (sskpd >> 56) & 0xFF;
2919                 if (wm[0] == 0)
2920                         wm[0] = sskpd & 0xF;
2921                 wm[1] = (sskpd >> 4) & 0xFF;
2922                 wm[2] = (sskpd >> 12) & 0xFF;
2923                 wm[3] = (sskpd >> 20) & 0x1FF;
2924                 wm[4] = (sskpd >> 32) & 0x1FF;
2925         } else if (INTEL_GEN(dev_priv) >= 6) {
2926                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2927
2928                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2929                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2930                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2931                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2932         } else if (INTEL_GEN(dev_priv) >= 5) {
2933                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2934
2935                 /* ILK primary LP0 latency is 700 ns */
2936                 wm[0] = 7;
2937                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2938                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2939         } else {
2940                 MISSING_CASE(INTEL_DEVID(dev_priv));
2941         }
2942 }
2943
2944 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2945                                        u16 wm[5])
2946 {
2947         /* ILK sprite LP0 latency is 1300 ns */
2948         if (IS_GEN(dev_priv, 5))
2949                 wm[0] = 13;
2950 }
2951
2952 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2953                                        u16 wm[5])
2954 {
2955         /* ILK cursor LP0 latency is 1300 ns */
2956         if (IS_GEN(dev_priv, 5))
2957                 wm[0] = 13;
2958 }
2959
2960 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2961 {
2962         /* how many WM levels are we expecting */
2963         if (INTEL_GEN(dev_priv) >= 9)
2964                 return 7;
2965         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2966                 return 4;
2967         else if (INTEL_GEN(dev_priv) >= 6)
2968                 return 3;
2969         else
2970                 return 2;
2971 }
2972
2973 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2974                                    const char *name,
2975                                    const u16 wm[8])
2976 {
2977         int level, max_level = ilk_wm_max_level(dev_priv);
2978
2979         for (level = 0; level <= max_level; level++) {
2980                 unsigned int latency = wm[level];
2981
2982                 if (latency == 0) {
2983                         drm_dbg_kms(&dev_priv->drm,
2984                                     "%s WM%d latency not provided\n",
2985                                     name, level);
2986                         continue;
2987                 }
2988
2989                 /*
2990                  * - latencies are in us on gen9.
2991                  * - before then, WM1+ latency values are in 0.5us units
2992                  */
2993                 if (INTEL_GEN(dev_priv) >= 9)
2994                         latency *= 10;
2995                 else if (level > 0)
2996                         latency *= 5;
2997
2998                 drm_dbg_kms(&dev_priv->drm,
2999                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
3000                             wm[level], latency / 10, latency % 10);
3001         }
3002 }
3003
3004 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3005                                     u16 wm[5], u16 min)
3006 {
3007         int level, max_level = ilk_wm_max_level(dev_priv);
3008
3009         if (wm[0] >= min)
3010                 return false;
3011
3012         wm[0] = max(wm[0], min);
3013         for (level = 1; level <= max_level; level++)
3014                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3015
3016         return true;
3017 }
3018
3019 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3020 {
3021         bool changed;
3022
3023         /*
3024          * The BIOS provided WM memory latency values are often
3025          * inadequate for high resolution displays. Adjust them.
3026          */
3027         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3028                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3029                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3030
3031         if (!changed)
3032                 return;
3033
3034         drm_dbg_kms(&dev_priv->drm,
3035                     "WM latency values increased to avoid potential underruns\n");
3036         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3037         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3038         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3039 }
3040
3041 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3042 {
3043         /*
3044          * On some SNB machines (Thinkpad X220 Tablet at least)
3045          * LP3 usage can cause vblank interrupts to be lost.
3046          * The DEIIR bit will go high but it looks like the CPU
3047          * never gets interrupted.
3048          *
3049          * It's not clear whether other interrupt source could
3050          * be affected or if this is somehow limited to vblank
3051          * interrupts only. To play it safe we disable LP3
3052          * watermarks entirely.
3053          */
3054         if (dev_priv->wm.pri_latency[3] == 0 &&
3055             dev_priv->wm.spr_latency[3] == 0 &&
3056             dev_priv->wm.cur_latency[3] == 0)
3057                 return;
3058
3059         dev_priv->wm.pri_latency[3] = 0;
3060         dev_priv->wm.spr_latency[3] = 0;
3061         dev_priv->wm.cur_latency[3] = 0;
3062
3063         drm_dbg_kms(&dev_priv->drm,
3064                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3065         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3066         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3067         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3068 }
3069
3070 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3071 {
3072         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3073
3074         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3075                sizeof(dev_priv->wm.pri_latency));
3076         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3077                sizeof(dev_priv->wm.pri_latency));
3078
3079         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3080         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3081
3082         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3083         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3084         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3085
3086         if (IS_GEN(dev_priv, 6)) {
3087                 snb_wm_latency_quirk(dev_priv);
3088                 snb_wm_lp3_irq_quirk(dev_priv);
3089         }
3090 }
3091
3092 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3093 {
3094         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3095         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3096 }
3097
3098 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3099                                  struct intel_pipe_wm *pipe_wm)
3100 {
3101         /* LP0 watermark maximums depend on this pipe alone */
3102         const struct intel_wm_config config = {
3103                 .num_pipes_active = 1,
3104                 .sprites_enabled = pipe_wm->sprites_enabled,
3105                 .sprites_scaled = pipe_wm->sprites_scaled,
3106         };
3107         struct ilk_wm_maximums max;
3108
3109         /* LP0 watermarks always use 1/2 DDB partitioning */
3110         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3111
3112         /* At least LP0 must be valid */
3113         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3114                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3115                 return false;
3116         }
3117
3118         return true;
3119 }
3120
3121 /* Compute new watermarks for the pipe */
3122 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3123 {
3124         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3125         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3126         struct intel_pipe_wm *pipe_wm;
3127         struct intel_plane *plane;
3128         const struct intel_plane_state *plane_state;
3129         const struct intel_plane_state *pristate = NULL;
3130         const struct intel_plane_state *sprstate = NULL;
3131         const struct intel_plane_state *curstate = NULL;
3132         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3133         struct ilk_wm_maximums max;
3134
3135         pipe_wm = &crtc_state->wm.ilk.optimal;
3136
3137         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3138                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3139                         pristate = plane_state;
3140                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3141                         sprstate = plane_state;
3142                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3143                         curstate = plane_state;
3144         }
3145
3146         pipe_wm->pipe_enabled = crtc_state->hw.active;
3147         if (sprstate) {
3148                 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3149                 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3150                         (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3151                          drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3152         }
3153
3154         usable_level = max_level;
3155
3156         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3157         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3158                 usable_level = 1;
3159
3160         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3161         if (pipe_wm->sprites_scaled)
3162                 usable_level = 0;
3163
3164         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3165         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3166                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3167
3168         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3169                 return -EINVAL;
3170
3171         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3172
3173         for (level = 1; level <= usable_level; level++) {
3174                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3175
3176                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3177                                      pristate, sprstate, curstate, wm);
3178
3179                 /*
3180                  * Disable any watermark level that exceeds the
3181                  * register maximums since such watermarks are
3182                  * always invalid.
3183                  */
3184                 if (!ilk_validate_wm_level(level, &max, wm)) {
3185                         memset(wm, 0, sizeof(*wm));
3186                         break;
3187                 }
3188         }
3189
3190         return 0;
3191 }
3192
3193 /*
3194  * Build a set of 'intermediate' watermark values that satisfy both the old
3195  * state and the new state.  These can be programmed to the hardware
3196  * immediately.
3197  */
3198 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3199 {
3200         struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3201         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3202         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3203         struct intel_atomic_state *intel_state =
3204                 to_intel_atomic_state(newstate->uapi.state);
3205         const struct intel_crtc_state *oldstate =
3206                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3207         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3208         int level, max_level = ilk_wm_max_level(dev_priv);
3209
3210         /*
3211          * Start with the final, target watermarks, then combine with the
3212          * currently active watermarks to get values that are safe both before
3213          * and after the vblank.
3214          */
3215         *a = newstate->wm.ilk.optimal;
3216         if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3217             intel_state->skip_intermediate_wm)
3218                 return 0;
3219
3220         a->pipe_enabled |= b->pipe_enabled;
3221         a->sprites_enabled |= b->sprites_enabled;
3222         a->sprites_scaled |= b->sprites_scaled;
3223
3224         for (level = 0; level <= max_level; level++) {
3225                 struct intel_wm_level *a_wm = &a->wm[level];
3226                 const struct intel_wm_level *b_wm = &b->wm[level];
3227
3228                 a_wm->enable &= b_wm->enable;
3229                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3230                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3231                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3232                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3233         }
3234
3235         /*
3236          * We need to make sure that these merged watermark values are
3237          * actually a valid configuration themselves.  If they're not,
3238          * there's no safe way to transition from the old state to
3239          * the new state, so we need to fail the atomic transaction.
3240          */
3241         if (!ilk_validate_pipe_wm(dev_priv, a))
3242                 return -EINVAL;
3243
3244         /*
3245          * If our intermediate WM are identical to the final WM, then we can
3246          * omit the post-vblank programming; only update if it's different.
3247          */
3248         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3249                 newstate->wm.need_postvbl_update = true;
3250
3251         return 0;
3252 }
3253
3254 /*
3255  * Merge the watermarks from all active pipes for a specific level.
3256  */
3257 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3258                                int level,
3259                                struct intel_wm_level *ret_wm)
3260 {
3261         const struct intel_crtc *intel_crtc;
3262
3263         ret_wm->enable = true;
3264
3265         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3266                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3267                 const struct intel_wm_level *wm = &active->wm[level];
3268
3269                 if (!active->pipe_enabled)
3270                         continue;
3271
3272                 /*
3273                  * The watermark values may have been used in the past,
3274                  * so we must maintain them in the registers for some
3275                  * time even if the level is now disabled.
3276                  */
3277                 if (!wm->enable)
3278                         ret_wm->enable = false;
3279
3280                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3281                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3282                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3283                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3284         }
3285 }
3286
3287 /*
3288  * Merge all low power watermarks for all active pipes.
3289  */
3290 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3291                          const struct intel_wm_config *config,
3292                          const struct ilk_wm_maximums *max,
3293                          struct intel_pipe_wm *merged)
3294 {
3295         int level, max_level = ilk_wm_max_level(dev_priv);
3296         int last_enabled_level = max_level;
3297
3298         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3299         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3300             config->num_pipes_active > 1)
3301                 last_enabled_level = 0;
3302
3303         /* ILK: FBC WM must be disabled always */
3304         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3305
3306         /* merge each WM1+ level */
3307         for (level = 1; level <= max_level; level++) {
3308                 struct intel_wm_level *wm = &merged->wm[level];
3309
3310                 ilk_merge_wm_level(dev_priv, level, wm);
3311
3312                 if (level > last_enabled_level)
3313                         wm->enable = false;
3314                 else if (!ilk_validate_wm_level(level, max, wm))
3315                         /* make sure all following levels get disabled */
3316                         last_enabled_level = level - 1;
3317
3318                 /*
3319                  * The spec says it is preferred to disable
3320                  * FBC WMs instead of disabling a WM level.
3321                  */
3322                 if (wm->fbc_val > max->fbc) {
3323                         if (wm->enable)
3324                                 merged->fbc_wm_enabled = false;
3325                         wm->fbc_val = 0;
3326                 }
3327         }
3328
3329         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3330         /*
3331          * FIXME this is racy. FBC might get enabled later.
3332          * What we should check here is whether FBC can be
3333          * enabled sometime later.
3334          */
3335         if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3336             intel_fbc_is_active(dev_priv)) {
3337                 for (level = 2; level <= max_level; level++) {
3338                         struct intel_wm_level *wm = &merged->wm[level];
3339
3340                         wm->enable = false;
3341                 }
3342         }
3343 }
3344
3345 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3346 {
3347         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3348         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3349 }
3350
3351 /* The value we need to program into the WM_LPx latency field */
3352 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3353                                       int level)
3354 {
3355         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3356                 return 2 * level;
3357         else
3358                 return dev_priv->wm.pri_latency[level];
3359 }
3360
3361 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3362                                    const struct intel_pipe_wm *merged,
3363                                    enum intel_ddb_partitioning partitioning,
3364                                    struct ilk_wm_values *results)
3365 {
3366         struct intel_crtc *intel_crtc;
3367         int level, wm_lp;
3368
3369         results->enable_fbc_wm = merged->fbc_wm_enabled;
3370         results->partitioning = partitioning;
3371
3372         /* LP1+ register values */
3373         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3374                 const struct intel_wm_level *r;
3375
3376                 level = ilk_wm_lp_to_level(wm_lp, merged);
3377
3378                 r = &merged->wm[level];
3379
3380                 /*
3381                  * Maintain the watermark values even if the level is
3382                  * disabled. Doing otherwise could cause underruns.
3383                  */
3384                 results->wm_lp[wm_lp - 1] =
3385                         (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3386                         (r->pri_val << WM1_LP_SR_SHIFT) |
3387                         r->cur_val;
3388
3389                 if (r->enable)
3390                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3391
3392                 if (INTEL_GEN(dev_priv) >= 8)
3393                         results->wm_lp[wm_lp - 1] |=
3394                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3395                 else
3396                         results->wm_lp[wm_lp - 1] |=
3397                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3398
3399                 /*
3400                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3401                  * level is disabled. Doing otherwise could cause underruns.
3402                  */
3403                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3404                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3405                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3406                 } else
3407                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3408         }
3409
3410         /* LP0 register values */
3411         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3412                 enum pipe pipe = intel_crtc->pipe;
3413                 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3414                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3415
3416                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3417                         continue;
3418
3419                 results->wm_pipe[pipe] =
3420                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3421                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3422                         r->cur_val;
3423         }
3424 }
3425
3426 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3427  * case both are at the same level. Prefer r1 in case they're the same. */
3428 static struct intel_pipe_wm *
3429 ilk_find_best_result(struct drm_i915_private *dev_priv,
3430                      struct intel_pipe_wm *r1,
3431                      struct intel_pipe_wm *r2)
3432 {
3433         int level, max_level = ilk_wm_max_level(dev_priv);
3434         int level1 = 0, level2 = 0;
3435
3436         for (level = 1; level <= max_level; level++) {
3437                 if (r1->wm[level].enable)
3438                         level1 = level;
3439                 if (r2->wm[level].enable)
3440                         level2 = level;
3441         }
3442
3443         if (level1 == level2) {
3444                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3445                         return r2;
3446                 else
3447                         return r1;
3448         } else if (level1 > level2) {
3449                 return r1;
3450         } else {
3451                 return r2;
3452         }
3453 }
3454
3455 /* dirty bits used to track which watermarks need changes */
3456 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3457 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3458 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3459 #define WM_DIRTY_FBC (1 << 24)
3460 #define WM_DIRTY_DDB (1 << 25)
3461
3462 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3463                                          const struct ilk_wm_values *old,
3464                                          const struct ilk_wm_values *new)
3465 {
3466         unsigned int dirty = 0;
3467         enum pipe pipe;
3468         int wm_lp;
3469
3470         for_each_pipe(dev_priv, pipe) {
3471                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3472                         dirty |= WM_DIRTY_PIPE(pipe);
3473                         /* Must disable LP1+ watermarks too */
3474                         dirty |= WM_DIRTY_LP_ALL;
3475                 }
3476         }
3477
3478         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3479                 dirty |= WM_DIRTY_FBC;
3480                 /* Must disable LP1+ watermarks too */
3481                 dirty |= WM_DIRTY_LP_ALL;
3482         }
3483
3484         if (old->partitioning != new->partitioning) {
3485                 dirty |= WM_DIRTY_DDB;
3486                 /* Must disable LP1+ watermarks too */
3487                 dirty |= WM_DIRTY_LP_ALL;
3488         }
3489
3490         /* LP1+ watermarks already deemed dirty, no need to continue */
3491         if (dirty & WM_DIRTY_LP_ALL)
3492                 return dirty;
3493
3494         /* Find the lowest numbered LP1+ watermark in need of an update... */
3495         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3496                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3497                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3498                         break;
3499         }
3500
3501         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3502         for (; wm_lp <= 3; wm_lp++)
3503                 dirty |= WM_DIRTY_LP(wm_lp);
3504
3505         return dirty;
3506 }
3507
3508 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3509                                unsigned int dirty)
3510 {
3511         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3512         bool changed = false;
3513
3514         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3515                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3516                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3517                 changed = true;
3518         }
3519         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3520                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3521                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3522                 changed = true;
3523         }
3524         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3525                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3526                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3527                 changed = true;
3528         }
3529
3530         /*
3531          * Don't touch WM1S_LP_EN here.
3532          * Doing so could cause underruns.
3533          */
3534
3535         return changed;
3536 }
3537
3538 /*
3539  * The spec says we shouldn't write when we don't need, because every write
3540  * causes WMs to be re-evaluated, expending some power.
3541  */
3542 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3543                                 struct ilk_wm_values *results)
3544 {
3545         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3546         unsigned int dirty;
3547         u32 val;
3548
3549         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3550         if (!dirty)
3551                 return;
3552
3553         _ilk_disable_lp_wm(dev_priv, dirty);
3554
3555         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3556                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3557         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3558                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3559         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3560                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3561
3562         if (dirty & WM_DIRTY_DDB) {
3563                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3564                         val = I915_READ(WM_MISC);
3565                         if (results->partitioning == INTEL_DDB_PART_1_2)
3566                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3567                         else
3568                                 val |= WM_MISC_DATA_PARTITION_5_6;
3569                         I915_WRITE(WM_MISC, val);
3570                 } else {
3571                         val = I915_READ(DISP_ARB_CTL2);
3572                         if (results->partitioning == INTEL_DDB_PART_1_2)
3573                                 val &= ~DISP_DATA_PARTITION_5_6;
3574                         else
3575                                 val |= DISP_DATA_PARTITION_5_6;
3576                         I915_WRITE(DISP_ARB_CTL2, val);
3577                 }
3578         }
3579
3580         if (dirty & WM_DIRTY_FBC) {
3581                 val = I915_READ(DISP_ARB_CTL);
3582                 if (results->enable_fbc_wm)
3583                         val &= ~DISP_FBC_WM_DIS;
3584                 else
3585                         val |= DISP_FBC_WM_DIS;
3586                 I915_WRITE(DISP_ARB_CTL, val);
3587         }
3588
3589         if (dirty & WM_DIRTY_LP(1) &&
3590             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3591                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3592
3593         if (INTEL_GEN(dev_priv) >= 7) {
3594                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3595                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3596                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3597                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3598         }
3599
3600         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3601                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3602         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3603                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3604         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3605                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3606
3607         dev_priv->wm.hw = *results;
3608 }
3609
3610 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3611 {
3612         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3613 }
3614
3615 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3616 {
3617         int i;
3618         int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3619         u8 enabled_slices_mask = 0;
3620
3621         for (i = 0; i < max_slices; i++) {
3622                 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3623                         enabled_slices_mask |= BIT(i);
3624         }
3625
3626         return enabled_slices_mask;
3627 }
3628
3629 /*
3630  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3631  * so assume we'll always need it in order to avoid underruns.
3632  */
3633 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3634 {
3635         return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3636 }
3637
3638 static bool
3639 intel_has_sagv(struct drm_i915_private *dev_priv)
3640 {
3641         return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3642                 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3643 }
3644
3645 static void
3646 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3647 {
3648         if (INTEL_GEN(dev_priv) >= 12) {
3649                 u32 val = 0;
3650                 int ret;
3651
3652                 ret = sandybridge_pcode_read(dev_priv,
3653                                              GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3654                                              &val, NULL);
3655                 if (!ret) {
3656                         dev_priv->sagv_block_time_us = val;
3657                         return;
3658                 }
3659
3660                 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3661         } else if (IS_GEN(dev_priv, 11)) {
3662                 dev_priv->sagv_block_time_us = 10;
3663                 return;
3664         } else if (IS_GEN(dev_priv, 10)) {
3665                 dev_priv->sagv_block_time_us = 20;
3666                 return;
3667         } else if (IS_GEN(dev_priv, 9)) {
3668                 dev_priv->sagv_block_time_us = 30;
3669                 return;
3670         } else {
3671                 MISSING_CASE(INTEL_GEN(dev_priv));
3672         }
3673
3674         /* Default to an unusable block time */
3675         dev_priv->sagv_block_time_us = -1;
3676 }
3677
3678 /*
3679  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3680  * depending on power and performance requirements. The display engine access
3681  * to system memory is blocked during the adjustment time. Because of the
3682  * blocking time, having this enabled can cause full system hangs and/or pipe
3683  * underruns if we don't meet all of the following requirements:
3684  *
3685  *  - <= 1 pipe enabled
3686  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3687  *  - We're not using an interlaced display configuration
3688  */
3689 int
3690 intel_enable_sagv(struct drm_i915_private *dev_priv)
3691 {
3692         int ret;
3693
3694         if (!intel_has_sagv(dev_priv))
3695                 return 0;
3696
3697         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3698                 return 0;
3699
3700         drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3701         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3702                                       GEN9_SAGV_ENABLE);
3703
3704         /* We don't need to wait for SAGV when enabling */
3705
3706         /*
3707          * Some skl systems, pre-release machines in particular,
3708          * don't actually have SAGV.
3709          */
3710         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3711                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3712                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3713                 return 0;
3714         } else if (ret < 0) {
3715                 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3716                 return ret;
3717         }
3718
3719         dev_priv->sagv_status = I915_SAGV_ENABLED;
3720         return 0;
3721 }
3722
3723 int
3724 intel_disable_sagv(struct drm_i915_private *dev_priv)
3725 {
3726         int ret;
3727
3728         if (!intel_has_sagv(dev_priv))
3729                 return 0;
3730
3731         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3732                 return 0;
3733
3734         drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3735         /* bspec says to keep retrying for at least 1 ms */
3736         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3737                                 GEN9_SAGV_DISABLE,
3738                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3739                                 1);
3740         /*
3741          * Some skl systems, pre-release machines in particular,
3742          * don't actually have SAGV.
3743          */
3744         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3745                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3746                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3747                 return 0;
3748         } else if (ret < 0) {
3749                 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3750                 return ret;
3751         }
3752
3753         dev_priv->sagv_status = I915_SAGV_DISABLED;
3754         return 0;
3755 }
3756
3757 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3758 {
3759         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3760         const struct intel_bw_state *new_bw_state;
3761         const struct intel_bw_state *old_bw_state;
3762         u32 new_mask = 0;
3763
3764         /*
3765          * Just return if we can't control SAGV or don't have it.
3766          * This is different from situation when we have SAGV but just can't
3767          * afford it due to DBuf limitation - in case if SAGV is completely
3768          * disabled in a BIOS, we are not even allowed to send a PCode request,
3769          * as it will throw an error. So have to check it here.
3770          */
3771         if (!intel_has_sagv(dev_priv))
3772                 return;
3773
3774         new_bw_state = intel_atomic_get_new_bw_state(state);
3775         if (!new_bw_state)
3776                 return;
3777
3778         if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3779                 intel_disable_sagv(dev_priv);
3780                 return;
3781         }
3782
3783         old_bw_state = intel_atomic_get_old_bw_state(state);
3784         /*
3785          * Nothing to mask
3786          */
3787         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3788                 return;
3789
3790         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3791
3792         /*
3793          * If new mask is zero - means there is nothing to mask,
3794          * we can only unmask, which should be done in unmask.
3795          */
3796         if (!new_mask)
3797                 return;
3798
3799         /*
3800          * Restrict required qgv points before updating the configuration.
3801          * According to BSpec we can't mask and unmask qgv points at the same
3802          * time. Also masking should be done before updating the configuration
3803          * and unmasking afterwards.
3804          */
3805         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3806 }
3807
3808 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3809 {
3810         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3811         const struct intel_bw_state *new_bw_state;
3812         const struct intel_bw_state *old_bw_state;
3813         u32 new_mask = 0;
3814
3815         /*
3816          * Just return if we can't control SAGV or don't have it.
3817          * This is different from situation when we have SAGV but just can't
3818          * afford it due to DBuf limitation - in case if SAGV is completely
3819          * disabled in a BIOS, we are not even allowed to send a PCode request,
3820          * as it will throw an error. So have to check it here.
3821          */
3822         if (!intel_has_sagv(dev_priv))
3823                 return;
3824
3825         new_bw_state = intel_atomic_get_new_bw_state(state);
3826         if (!new_bw_state)
3827                 return;
3828
3829         if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3830                 intel_enable_sagv(dev_priv);
3831                 return;
3832         }
3833
3834         old_bw_state = intel_atomic_get_old_bw_state(state);
3835         /*
3836          * Nothing to unmask
3837          */
3838         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3839                 return;
3840
3841         new_mask = new_bw_state->qgv_points_mask;
3842
3843         /*
3844          * Allow required qgv points after updating the configuration.
3845          * According to BSpec we can't mask and unmask qgv points at the same
3846          * time. Also masking should be done before updating the configuration
3847          * and unmasking afterwards.
3848          */
3849         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3850 }
3851
3852 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3853 {
3854         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3855         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3856         struct intel_plane *plane;
3857         const struct intel_plane_state *plane_state;
3858         int level, latency;
3859
3860         if (!intel_has_sagv(dev_priv))
3861                 return false;
3862
3863         if (!crtc_state->hw.active)
3864                 return true;
3865
3866         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3867                 return false;
3868
3869         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3870                 const struct skl_plane_wm *wm =
3871                         &crtc_state->wm.skl.optimal.planes[plane->id];
3872
3873                 /* Skip this plane if it's not enabled */
3874                 if (!wm->wm[0].plane_en)
3875                         continue;
3876
3877                 /* Find the highest enabled wm level for this plane */
3878                 for (level = ilk_wm_max_level(dev_priv);
3879                      !wm->wm[level].plane_en; --level)
3880                      { }
3881
3882                 latency = dev_priv->wm.skl_latency[level];
3883
3884                 if (skl_needs_memory_bw_wa(dev_priv) &&
3885                     plane_state->uapi.fb->modifier ==
3886                     I915_FORMAT_MOD_X_TILED)
3887                         latency += 15;
3888
3889                 /*
3890                  * If any of the planes on this pipe don't enable wm levels that
3891                  * incur memory latencies higher than sagv_block_time_us we
3892                  * can't enable SAGV.
3893                  */
3894                 if (latency < dev_priv->sagv_block_time_us)
3895                         return false;
3896         }
3897
3898         return true;
3899 }
3900
3901 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3902 {
3903         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3904         enum plane_id plane_id;
3905
3906         if (!crtc_state->hw.active)
3907                 return true;
3908
3909         for_each_plane_id_on_crtc(crtc, plane_id) {
3910                 const struct skl_ddb_entry *plane_alloc =
3911                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
3912                 const struct skl_plane_wm *wm =
3913                         &crtc_state->wm.skl.optimal.planes[plane_id];
3914
3915                 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3916                         return false;
3917         }
3918
3919         return true;
3920 }
3921
3922 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3923 {
3924         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3925         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3926
3927         if (INTEL_GEN(dev_priv) >= 12)
3928                 return tgl_crtc_can_enable_sagv(crtc_state);
3929         else
3930                 return skl_crtc_can_enable_sagv(crtc_state);
3931 }
3932
3933 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3934                            const struct intel_bw_state *bw_state)
3935 {
3936         if (INTEL_GEN(dev_priv) < 11 &&
3937             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3938                 return false;
3939
3940         return bw_state->pipe_sagv_reject == 0;
3941 }
3942
3943 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3944 {
3945         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3946         int ret;
3947         struct intel_crtc *crtc;
3948         struct intel_crtc_state *new_crtc_state;
3949         struct intel_bw_state *new_bw_state = NULL;
3950         const struct intel_bw_state *old_bw_state = NULL;
3951         int i;
3952
3953         for_each_new_intel_crtc_in_state(state, crtc,
3954                                          new_crtc_state, i) {
3955                 new_bw_state = intel_atomic_get_bw_state(state);
3956                 if (IS_ERR(new_bw_state))
3957                         return PTR_ERR(new_bw_state);
3958
3959                 old_bw_state = intel_atomic_get_old_bw_state(state);
3960
3961                 if (intel_crtc_can_enable_sagv(new_crtc_state))
3962                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3963                 else
3964                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3965         }
3966
3967         if (!new_bw_state)
3968                 return 0;
3969
3970         new_bw_state->active_pipes =
3971                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3972
3973         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3974                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3975                 if (ret)
3976                         return ret;
3977         }
3978
3979         for_each_new_intel_crtc_in_state(state, crtc,
3980                                          new_crtc_state, i) {
3981                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
3982
3983                 /*
3984                  * We store use_sagv_wm in the crtc state rather than relying on
3985                  * that bw state since we have no convenient way to get at the
3986                  * latter from the plane commit hooks (especially in the legacy
3987                  * cursor case)
3988                  */
3989                 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
3990                                        intel_can_enable_sagv(dev_priv, new_bw_state);
3991         }
3992
3993         if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
3994             intel_can_enable_sagv(dev_priv, old_bw_state)) {
3995                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
3996                 if (ret)
3997                         return ret;
3998         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
3999                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4000                 if (ret)
4001                         return ret;
4002         }
4003
4004         return 0;
4005 }
4006
4007 /*
4008  * Calculate initial DBuf slice offset, based on slice size
4009  * and mask(i.e if slice size is 1024 and second slice is enabled
4010  * offset would be 1024)
4011  */
4012 static unsigned int
4013 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4014                                 u32 slice_size,
4015                                 u32 ddb_size)
4016 {
4017         unsigned int offset = 0;
4018
4019         if (!dbuf_slice_mask)
4020                 return 0;
4021
4022         offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4023
4024         WARN_ON(offset >= ddb_size);
4025         return offset;
4026 }
4027
4028 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4029 {
4030         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4031
4032         drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4033
4034         if (INTEL_GEN(dev_priv) < 11)
4035                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4036
4037         return ddb_size;
4038 }
4039
4040 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4041                                   u8 active_pipes);
4042
4043 static void
4044 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4045                                    const struct intel_crtc_state *crtc_state,
4046                                    const u64 total_data_rate,
4047                                    struct skl_ddb_entry *alloc, /* out */
4048                                    int *num_active /* out */)
4049 {
4050         struct drm_atomic_state *state = crtc_state->uapi.state;
4051         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4052         struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4053         const struct intel_crtc *crtc;
4054         u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4055         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4056         u16 ddb_size;
4057         u32 ddb_range_size;
4058         u32 i;
4059         u32 dbuf_slice_mask;
4060         u32 active_pipes;
4061         u32 offset;
4062         u32 slice_size;
4063         u32 total_slice_mask;
4064         u32 start, end;
4065
4066         if (drm_WARN_ON(&dev_priv->drm, !state) || !crtc_state->hw.active) {
4067                 alloc->start = 0;
4068                 alloc->end = 0;
4069                 *num_active = hweight8(dev_priv->active_pipes);
4070                 return;
4071         }
4072
4073         if (intel_state->active_pipe_changes)
4074                 active_pipes = intel_state->active_pipes;
4075         else
4076                 active_pipes = dev_priv->active_pipes;
4077
4078         *num_active = hweight8(active_pipes);
4079
4080         ddb_size = intel_get_ddb_size(dev_priv);
4081
4082         slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4083
4084         /*
4085          * If the state doesn't change the active CRTC's or there is no
4086          * modeset request, then there's no need to recalculate;
4087          * the existing pipe allocation limits should remain unchanged.
4088          * Note that we're safe from racing commits since any racing commit
4089          * that changes the active CRTC list or do modeset would need to
4090          * grab _all_ crtc locks, including the one we currently hold.
4091          */
4092         if (!intel_state->active_pipe_changes && !intel_state->modeset) {
4093                 /*
4094                  * alloc may be cleared by clear_intel_crtc_state,
4095                  * copy from old state to be sure
4096                  */
4097                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4098                 return;
4099         }
4100
4101         /*
4102          * Get allowed DBuf slices for correspondent pipe and platform.
4103          */
4104         dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4105
4106         DRM_DEBUG_KMS("DBuf slice mask %x pipe %c active pipes %x\n",
4107                       dbuf_slice_mask,
4108                       pipe_name(for_pipe), active_pipes);
4109
4110         /*
4111          * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4112          * and slice size is 1024, the offset would be 1024
4113          */
4114         offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4115                                                  slice_size, ddb_size);
4116
4117         /*
4118          * Figure out total size of allowed DBuf slices, which is basically
4119          * a number of allowed slices for that pipe multiplied by slice size.
4120          * Inside of this
4121          * range ddb entries are still allocated in proportion to display width.
4122          */
4123         ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4124
4125         /*
4126          * Watermark/ddb requirement highly depends upon width of the
4127          * framebuffer, So instead of allocating DDB equally among pipes
4128          * distribute DDB based on resolution/width of the display.
4129          */
4130         total_slice_mask = dbuf_slice_mask;
4131         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4132                 const struct drm_display_mode *adjusted_mode =
4133                         &crtc_state->hw.adjusted_mode;
4134                 enum pipe pipe = crtc->pipe;
4135                 int hdisplay, vdisplay;
4136                 u32 pipe_dbuf_slice_mask;
4137
4138                 if (!crtc_state->hw.active)
4139                         continue;
4140
4141                 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4142                                                                active_pipes);
4143
4144                 /*
4145                  * According to BSpec pipe can share one dbuf slice with another
4146                  * pipes or pipe can use multiple dbufs, in both cases we
4147                  * account for other pipes only if they have exactly same mask.
4148                  * However we need to account how many slices we should enable
4149                  * in total.
4150                  */
4151                 total_slice_mask |= pipe_dbuf_slice_mask;
4152
4153                 /*
4154                  * Do not account pipes using other slice sets
4155                  * luckily as of current BSpec slice sets do not partially
4156                  * intersect(pipes share either same one slice or same slice set
4157                  * i.e no partial intersection), so it is enough to check for
4158                  * equality for now.
4159                  */
4160                 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4161                         continue;
4162
4163                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4164
4165                 total_width_in_range += hdisplay;
4166
4167                 if (pipe < for_pipe)
4168                         width_before_pipe_in_range += hdisplay;
4169                 else if (pipe == for_pipe)
4170                         pipe_width = hdisplay;
4171         }
4172
4173         /*
4174          * FIXME: For now we always enable slice S1 as per
4175          * the Bspec display initialization sequence.
4176          */
4177         intel_state->enabled_dbuf_slices_mask = total_slice_mask | BIT(DBUF_S1);
4178
4179         start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4180         end = ddb_range_size *
4181                 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4182
4183         alloc->start = offset + start;
4184         alloc->end = offset + end;
4185
4186         DRM_DEBUG_KMS("Pipe %d ddb %d-%d\n", for_pipe,
4187                       alloc->start, alloc->end);
4188         DRM_DEBUG_KMS("Enabled ddb slices mask %x num supported %d\n",
4189                       intel_state->enabled_dbuf_slices_mask,
4190                       INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
4191 }
4192
4193 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4194                                  int width, const struct drm_format_info *format,
4195                                  u64 modifier, unsigned int rotation,
4196                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
4197                                  int color_plane);
4198 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4199                                  int level,
4200                                  unsigned int latency,
4201                                  const struct skl_wm_params *wp,
4202                                  const struct skl_wm_level *result_prev,
4203                                  struct skl_wm_level *result /* out */);
4204
4205 static unsigned int
4206 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4207                       int num_active)
4208 {
4209         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4210         int level, max_level = ilk_wm_max_level(dev_priv);
4211         struct skl_wm_level wm = {};
4212         int ret, min_ddb_alloc = 0;
4213         struct skl_wm_params wp;
4214
4215         ret = skl_compute_wm_params(crtc_state, 256,
4216                                     drm_format_info(DRM_FORMAT_ARGB8888),
4217                                     DRM_FORMAT_MOD_LINEAR,
4218                                     DRM_MODE_ROTATE_0,
4219                                     crtc_state->pixel_rate, &wp, 0);
4220         drm_WARN_ON(&dev_priv->drm, ret);
4221
4222         for (level = 0; level <= max_level; level++) {
4223                 unsigned int latency = dev_priv->wm.skl_latency[level];
4224
4225                 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4226                 if (wm.min_ddb_alloc == U16_MAX)
4227                         break;
4228
4229                 min_ddb_alloc = wm.min_ddb_alloc;
4230         }
4231
4232         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4233 }
4234
4235 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4236                                        struct skl_ddb_entry *entry, u32 reg)
4237 {
4238
4239         entry->start = reg & DDB_ENTRY_MASK;
4240         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4241
4242         if (entry->end)
4243                 entry->end += 1;
4244 }
4245
4246 static void
4247 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4248                            const enum pipe pipe,
4249                            const enum plane_id plane_id,
4250                            struct skl_ddb_entry *ddb_y,
4251                            struct skl_ddb_entry *ddb_uv)
4252 {
4253         u32 val, val2;
4254         u32 fourcc = 0;
4255
4256         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4257         if (plane_id == PLANE_CURSOR) {
4258                 val = I915_READ(CUR_BUF_CFG(pipe));
4259                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4260                 return;
4261         }
4262
4263         val = I915_READ(PLANE_CTL(pipe, plane_id));
4264
4265         /* No DDB allocated for disabled planes */
4266         if (val & PLANE_CTL_ENABLE)
4267                 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4268                                               val & PLANE_CTL_ORDER_RGBX,
4269                                               val & PLANE_CTL_ALPHA_MASK);
4270
4271         if (INTEL_GEN(dev_priv) >= 11) {
4272                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4273                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4274         } else {
4275                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4276                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4277
4278                 if (fourcc &&
4279                     drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4280                         swap(val, val2);
4281
4282                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4283                 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4284         }
4285 }
4286
4287 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4288                                struct skl_ddb_entry *ddb_y,
4289                                struct skl_ddb_entry *ddb_uv)
4290 {
4291         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4292         enum intel_display_power_domain power_domain;
4293         enum pipe pipe = crtc->pipe;
4294         intel_wakeref_t wakeref;
4295         enum plane_id plane_id;
4296
4297         power_domain = POWER_DOMAIN_PIPE(pipe);
4298         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4299         if (!wakeref)
4300                 return;
4301
4302         for_each_plane_id_on_crtc(crtc, plane_id)
4303                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4304                                            plane_id,
4305                                            &ddb_y[plane_id],
4306                                            &ddb_uv[plane_id]);
4307
4308         intel_display_power_put(dev_priv, power_domain, wakeref);
4309 }
4310
4311 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv)
4312 {
4313         dev_priv->enabled_dbuf_slices_mask =
4314                                 intel_enabled_dbuf_slices_mask(dev_priv);
4315 }
4316
4317 /*
4318  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4319  * The bspec defines downscale amount as:
4320  *
4321  * """
4322  * Horizontal down scale amount = maximum[1, Horizontal source size /
4323  *                                           Horizontal destination size]
4324  * Vertical down scale amount = maximum[1, Vertical source size /
4325  *                                         Vertical destination size]
4326  * Total down scale amount = Horizontal down scale amount *
4327  *                           Vertical down scale amount
4328  * """
4329  *
4330  * Return value is provided in 16.16 fixed point form to retain fractional part.
4331  * Caller should take care of dividing & rounding off the value.
4332  */
4333 static uint_fixed_16_16_t
4334 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4335                            const struct intel_plane_state *plane_state)
4336 {
4337         u32 src_w, src_h, dst_w, dst_h;
4338         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4339         uint_fixed_16_16_t downscale_h, downscale_w;
4340
4341         if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4342                 return u32_to_fixed16(0);
4343
4344         /*
4345          * Src coordinates are already rotated by 270 degrees for
4346          * the 90/270 degree plane rotation cases (to match the
4347          * GTT mapping), hence no need to account for rotation here.
4348          *
4349          * n.b., src is 16.16 fixed point, dst is whole integer.
4350          */
4351         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4352         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4353         dst_w = drm_rect_width(&plane_state->uapi.dst);
4354         dst_h = drm_rect_height(&plane_state->uapi.dst);
4355
4356         fp_w_ratio = div_fixed16(src_w, dst_w);
4357         fp_h_ratio = div_fixed16(src_h, dst_h);
4358         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4359         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4360
4361         return mul_fixed16(downscale_w, downscale_h);
4362 }
4363
4364 struct dbuf_slice_conf_entry {
4365         u8 active_pipes;
4366         u8 dbuf_mask[I915_MAX_PIPES];
4367 };
4368
4369 /*
4370  * Table taken from Bspec 12716
4371  * Pipes do have some preferred DBuf slice affinity,
4372  * plus there are some hardcoded requirements on how
4373  * those should be distributed for multipipe scenarios.
4374  * For more DBuf slices algorithm can get even more messy
4375  * and less readable, so decided to use a table almost
4376  * as is from BSpec itself - that way it is at least easier
4377  * to compare, change and check.
4378  */
4379 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4380 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4381 {
4382         {
4383                 .active_pipes = BIT(PIPE_A),
4384                 .dbuf_mask = {
4385                         [PIPE_A] = BIT(DBUF_S1),
4386                 },
4387         },
4388         {
4389                 .active_pipes = BIT(PIPE_B),
4390                 .dbuf_mask = {
4391                         [PIPE_B] = BIT(DBUF_S1),
4392                 },
4393         },
4394         {
4395                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4396                 .dbuf_mask = {
4397                         [PIPE_A] = BIT(DBUF_S1),
4398                         [PIPE_B] = BIT(DBUF_S2),
4399                 },
4400         },
4401         {
4402                 .active_pipes = BIT(PIPE_C),
4403                 .dbuf_mask = {
4404                         [PIPE_C] = BIT(DBUF_S2),
4405                 },
4406         },
4407         {
4408                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4409                 .dbuf_mask = {
4410                         [PIPE_A] = BIT(DBUF_S1),
4411                         [PIPE_C] = BIT(DBUF_S2),
4412                 },
4413         },
4414         {
4415                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4416                 .dbuf_mask = {
4417                         [PIPE_B] = BIT(DBUF_S1),
4418                         [PIPE_C] = BIT(DBUF_S2),
4419                 },
4420         },
4421         {
4422                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4423                 .dbuf_mask = {
4424                         [PIPE_A] = BIT(DBUF_S1),
4425                         [PIPE_B] = BIT(DBUF_S1),
4426                         [PIPE_C] = BIT(DBUF_S2),
4427                 },
4428         },
4429         {}
4430 };
4431
4432 /*
4433  * Table taken from Bspec 49255
4434  * Pipes do have some preferred DBuf slice affinity,
4435  * plus there are some hardcoded requirements on how
4436  * those should be distributed for multipipe scenarios.
4437  * For more DBuf slices algorithm can get even more messy
4438  * and less readable, so decided to use a table almost
4439  * as is from BSpec itself - that way it is at least easier
4440  * to compare, change and check.
4441  */
4442 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4443 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4444 {
4445         {
4446                 .active_pipes = BIT(PIPE_A),
4447                 .dbuf_mask = {
4448                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4449                 },
4450         },
4451         {
4452                 .active_pipes = BIT(PIPE_B),
4453                 .dbuf_mask = {
4454                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4455                 },
4456         },
4457         {
4458                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4459                 .dbuf_mask = {
4460                         [PIPE_A] = BIT(DBUF_S2),
4461                         [PIPE_B] = BIT(DBUF_S1),
4462                 },
4463         },
4464         {
4465                 .active_pipes = BIT(PIPE_C),
4466                 .dbuf_mask = {
4467                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4468                 },
4469         },
4470         {
4471                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4472                 .dbuf_mask = {
4473                         [PIPE_A] = BIT(DBUF_S1),
4474                         [PIPE_C] = BIT(DBUF_S2),
4475                 },
4476         },
4477         {
4478                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4479                 .dbuf_mask = {
4480                         [PIPE_B] = BIT(DBUF_S1),
4481                         [PIPE_C] = BIT(DBUF_S2),
4482                 },
4483         },
4484         {
4485                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4486                 .dbuf_mask = {
4487                         [PIPE_A] = BIT(DBUF_S1),
4488                         [PIPE_B] = BIT(DBUF_S1),
4489                         [PIPE_C] = BIT(DBUF_S2),
4490                 },
4491         },
4492         {
4493                 .active_pipes = BIT(PIPE_D),
4494                 .dbuf_mask = {
4495                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4496                 },
4497         },
4498         {
4499                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4500                 .dbuf_mask = {
4501                         [PIPE_A] = BIT(DBUF_S1),
4502                         [PIPE_D] = BIT(DBUF_S2),
4503                 },
4504         },
4505         {
4506                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4507                 .dbuf_mask = {
4508                         [PIPE_B] = BIT(DBUF_S1),
4509                         [PIPE_D] = BIT(DBUF_S2),
4510                 },
4511         },
4512         {
4513                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4514                 .dbuf_mask = {
4515                         [PIPE_A] = BIT(DBUF_S1),
4516                         [PIPE_B] = BIT(DBUF_S1),
4517                         [PIPE_D] = BIT(DBUF_S2),
4518                 },
4519         },
4520         {
4521                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4522                 .dbuf_mask = {
4523                         [PIPE_C] = BIT(DBUF_S1),
4524                         [PIPE_D] = BIT(DBUF_S2),
4525                 },
4526         },
4527         {
4528                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4529                 .dbuf_mask = {
4530                         [PIPE_A] = BIT(DBUF_S1),
4531                         [PIPE_C] = BIT(DBUF_S2),
4532                         [PIPE_D] = BIT(DBUF_S2),
4533                 },
4534         },
4535         {
4536                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4537                 .dbuf_mask = {
4538                         [PIPE_B] = BIT(DBUF_S1),
4539                         [PIPE_C] = BIT(DBUF_S2),
4540                         [PIPE_D] = BIT(DBUF_S2),
4541                 },
4542         },
4543         {
4544                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4545                 .dbuf_mask = {
4546                         [PIPE_A] = BIT(DBUF_S1),
4547                         [PIPE_B] = BIT(DBUF_S1),
4548                         [PIPE_C] = BIT(DBUF_S2),
4549                         [PIPE_D] = BIT(DBUF_S2),
4550                 },
4551         },
4552         {}
4553 };
4554
4555 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4556                               const struct dbuf_slice_conf_entry *dbuf_slices)
4557 {
4558         int i;
4559
4560         for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4561                 if (dbuf_slices[i].active_pipes == active_pipes)
4562                         return dbuf_slices[i].dbuf_mask[pipe];
4563         }
4564         return 0;
4565 }
4566
4567 /*
4568  * This function finds an entry with same enabled pipe configuration and
4569  * returns correspondent DBuf slice mask as stated in BSpec for particular
4570  * platform.
4571  */
4572 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4573 {
4574         /*
4575          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4576          * required calculating "pipe ratio" in order to determine
4577          * if one or two slices can be used for single pipe configurations
4578          * as additional constraint to the existing table.
4579          * However based on recent info, it should be not "pipe ratio"
4580          * but rather ratio between pixel_rate and cdclk with additional
4581          * constants, so for now we are using only table until this is
4582          * clarified. Also this is the reason why crtc_state param is
4583          * still here - we will need it once those additional constraints
4584          * pop up.
4585          */
4586         return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4587 }
4588
4589 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4590 {
4591         return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4592 }
4593
4594 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4595                                   u8 active_pipes)
4596 {
4597         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4598         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4599         enum pipe pipe = crtc->pipe;
4600
4601         if (IS_GEN(dev_priv, 12))
4602                 return tgl_compute_dbuf_slices(pipe, active_pipes);
4603         else if (IS_GEN(dev_priv, 11))
4604                 return icl_compute_dbuf_slices(pipe, active_pipes);
4605         /*
4606          * For anything else just return one slice yet.
4607          * Should be extended for other platforms.
4608          */
4609         return BIT(DBUF_S1);
4610 }
4611
4612 static u64
4613 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4614                              const struct intel_plane_state *plane_state,
4615                              int color_plane)
4616 {
4617         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4618         const struct drm_framebuffer *fb = plane_state->hw.fb;
4619         u32 data_rate;
4620         u32 width = 0, height = 0;
4621         uint_fixed_16_16_t down_scale_amount;
4622         u64 rate;
4623
4624         if (!plane_state->uapi.visible)
4625                 return 0;
4626
4627         if (plane->id == PLANE_CURSOR)
4628                 return 0;
4629
4630         if (color_plane == 1 &&
4631             !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4632                 return 0;
4633
4634         /*
4635          * Src coordinates are already rotated by 270 degrees for
4636          * the 90/270 degree plane rotation cases (to match the
4637          * GTT mapping), hence no need to account for rotation here.
4638          */
4639         width = drm_rect_width(&plane_state->uapi.src) >> 16;
4640         height = drm_rect_height(&plane_state->uapi.src) >> 16;
4641
4642         /* UV plane does 1/2 pixel sub-sampling */
4643         if (color_plane == 1) {
4644                 width /= 2;
4645                 height /= 2;
4646         }
4647
4648         data_rate = width * height;
4649
4650         down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4651
4652         rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4653
4654         rate *= fb->format->cpp[color_plane];
4655         return rate;
4656 }
4657
4658 static u64
4659 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4660                                  u64 *plane_data_rate,
4661                                  u64 *uv_plane_data_rate)
4662 {
4663         struct intel_plane *plane;
4664         const struct intel_plane_state *plane_state;
4665         u64 total_data_rate = 0;
4666
4667         /* Calculate and cache data rate for each plane */
4668         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4669                 enum plane_id plane_id = plane->id;
4670                 u64 rate;
4671
4672                 /* packed/y */
4673                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4674                 plane_data_rate[plane_id] = rate;
4675                 total_data_rate += rate;
4676
4677                 /* uv-plane */
4678                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4679                 uv_plane_data_rate[plane_id] = rate;
4680                 total_data_rate += rate;
4681         }
4682
4683         return total_data_rate;
4684 }
4685
4686 static u64
4687 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4688                                  u64 *plane_data_rate)
4689 {
4690         struct intel_plane *plane;
4691         const struct intel_plane_state *plane_state;
4692         u64 total_data_rate = 0;
4693
4694         /* Calculate and cache data rate for each plane */
4695         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4696                 enum plane_id plane_id = plane->id;
4697                 u64 rate;
4698
4699                 if (!plane_state->planar_linked_plane) {
4700                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4701                         plane_data_rate[plane_id] = rate;
4702                         total_data_rate += rate;
4703                 } else {
4704                         enum plane_id y_plane_id;
4705
4706                         /*
4707                          * The slave plane might not iterate in
4708                          * intel_atomic_crtc_state_for_each_plane_state(),
4709                          * and needs the master plane state which may be
4710                          * NULL if we try get_new_plane_state(), so we
4711                          * always calculate from the master.
4712                          */
4713                         if (plane_state->planar_slave)
4714                                 continue;
4715
4716                         /* Y plane rate is calculated on the slave */
4717                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4718                         y_plane_id = plane_state->planar_linked_plane->id;
4719                         plane_data_rate[y_plane_id] = rate;
4720                         total_data_rate += rate;
4721
4722                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4723                         plane_data_rate[plane_id] = rate;
4724                         total_data_rate += rate;
4725                 }
4726         }
4727
4728         return total_data_rate;
4729 }
4730
4731 static const struct skl_wm_level *
4732 skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4733                    enum plane_id plane_id,
4734                    int level)
4735 {
4736         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4737         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4738
4739         if (level == 0 && pipe_wm->use_sagv_wm)
4740                 return &wm->sagv_wm0;
4741
4742         return &wm->wm[level];
4743 }
4744
4745 static int
4746 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4747 {
4748         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4749         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4750         struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4751         u16 alloc_size, start = 0;
4752         u16 total[I915_MAX_PLANES] = {};
4753         u16 uv_total[I915_MAX_PLANES] = {};
4754         u64 total_data_rate;
4755         enum plane_id plane_id;
4756         int num_active;
4757         u64 plane_data_rate[I915_MAX_PLANES] = {};
4758         u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4759         u32 blocks;
4760         int level;
4761
4762         /* Clear the partitioning for disabled planes. */
4763         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4764         memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4765
4766         if (!crtc_state->hw.active) {
4767                 alloc->start = alloc->end = 0;
4768                 return 0;
4769         }
4770
4771         if (INTEL_GEN(dev_priv) >= 11)
4772                 total_data_rate =
4773                         icl_get_total_relative_data_rate(crtc_state,
4774                                                          plane_data_rate);
4775         else
4776                 total_data_rate =
4777                         skl_get_total_relative_data_rate(crtc_state,
4778                                                          plane_data_rate,
4779                                                          uv_plane_data_rate);
4780
4781         skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate,
4782                                            alloc, &num_active);
4783         alloc_size = skl_ddb_entry_size(alloc);
4784         if (alloc_size == 0)
4785                 return 0;
4786
4787         /* Allocate fixed number of blocks for cursor. */
4788         total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4789         alloc_size -= total[PLANE_CURSOR];
4790         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4791                 alloc->end - total[PLANE_CURSOR];
4792         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4793
4794         if (total_data_rate == 0)
4795                 return 0;
4796
4797         /*
4798          * Find the highest watermark level for which we can satisfy the block
4799          * requirement of active planes.
4800          */
4801         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4802                 blocks = 0;
4803                 for_each_plane_id_on_crtc(crtc, plane_id) {
4804                         const struct skl_plane_wm *wm =
4805                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4806
4807                         if (plane_id == PLANE_CURSOR) {
4808                                 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4809                                         drm_WARN_ON(&dev_priv->drm,
4810                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
4811                                         blocks = U32_MAX;
4812                                         break;
4813                                 }
4814                                 continue;
4815                         }
4816
4817                         blocks += wm->wm[level].min_ddb_alloc;
4818                         blocks += wm->uv_wm[level].min_ddb_alloc;
4819                 }
4820
4821                 if (blocks <= alloc_size) {
4822                         alloc_size -= blocks;
4823                         break;
4824                 }
4825         }
4826
4827         if (level < 0) {
4828                 drm_dbg_kms(&dev_priv->drm,
4829                             "Requested display configuration exceeds system DDB limitations");
4830                 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4831                             blocks, alloc_size);
4832                 return -EINVAL;
4833         }
4834
4835         /*
4836          * Grant each plane the blocks it requires at the highest achievable
4837          * watermark level, plus an extra share of the leftover blocks
4838          * proportional to its relative data rate.
4839          */
4840         for_each_plane_id_on_crtc(crtc, plane_id) {
4841                 const struct skl_plane_wm *wm =
4842                         &crtc_state->wm.skl.optimal.planes[plane_id];
4843                 u64 rate;
4844                 u16 extra;
4845
4846                 if (plane_id == PLANE_CURSOR)
4847                         continue;
4848
4849                 /*
4850                  * We've accounted for all active planes; remaining planes are
4851                  * all disabled.
4852                  */
4853                 if (total_data_rate == 0)
4854                         break;
4855
4856                 rate = plane_data_rate[plane_id];
4857                 extra = min_t(u16, alloc_size,
4858                               DIV64_U64_ROUND_UP(alloc_size * rate,
4859                                                  total_data_rate));
4860                 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4861                 alloc_size -= extra;
4862                 total_data_rate -= rate;
4863
4864                 if (total_data_rate == 0)
4865                         break;
4866
4867                 rate = uv_plane_data_rate[plane_id];
4868                 extra = min_t(u16, alloc_size,
4869                               DIV64_U64_ROUND_UP(alloc_size * rate,
4870                                                  total_data_rate));
4871                 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4872                 alloc_size -= extra;
4873                 total_data_rate -= rate;
4874         }
4875         drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4876
4877         /* Set the actual DDB start/end points for each plane */
4878         start = alloc->start;
4879         for_each_plane_id_on_crtc(crtc, plane_id) {
4880                 struct skl_ddb_entry *plane_alloc =
4881                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
4882                 struct skl_ddb_entry *uv_plane_alloc =
4883                         &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4884
4885                 if (plane_id == PLANE_CURSOR)
4886                         continue;
4887
4888                 /* Gen11+ uses a separate plane for UV watermarks */
4889                 drm_WARN_ON(&dev_priv->drm,
4890                             INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4891
4892                 /* Leave disabled planes at (0,0) */
4893                 if (total[plane_id]) {
4894                         plane_alloc->start = start;
4895                         start += total[plane_id];
4896                         plane_alloc->end = start;
4897                 }
4898
4899                 if (uv_total[plane_id]) {
4900                         uv_plane_alloc->start = start;
4901                         start += uv_total[plane_id];
4902                         uv_plane_alloc->end = start;
4903                 }
4904         }
4905
4906         /*
4907          * When we calculated watermark values we didn't know how high
4908          * of a level we'd actually be able to hit, so we just marked
4909          * all levels as "enabled."  Go back now and disable the ones
4910          * that aren't actually possible.
4911          */
4912         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4913                 for_each_plane_id_on_crtc(crtc, plane_id) {
4914                         struct skl_plane_wm *wm =
4915                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4916
4917                         /*
4918                          * We only disable the watermarks for each plane if
4919                          * they exceed the ddb allocation of said plane. This
4920                          * is done so that we don't end up touching cursor
4921                          * watermarks needlessly when some other plane reduces
4922                          * our max possible watermark level.
4923                          *
4924                          * Bspec has this to say about the PLANE_WM enable bit:
4925                          * "All the watermarks at this level for all enabled
4926                          *  planes must be enabled before the level will be used."
4927                          * So this is actually safe to do.
4928                          */
4929                         if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
4930                             wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
4931                                 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
4932
4933                         /*
4934                          * Wa_1408961008:icl, ehl
4935                          * Underruns with WM1+ disabled
4936                          */
4937                         if (IS_GEN(dev_priv, 11) &&
4938                             level == 1 && wm->wm[0].plane_en) {
4939                                 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
4940                                 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
4941                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4942                         }
4943                 }
4944         }
4945
4946         /*
4947          * Go back and disable the transition watermark if it turns out we
4948          * don't have enough DDB blocks for it.
4949          */
4950         for_each_plane_id_on_crtc(crtc, plane_id) {
4951                 struct skl_plane_wm *wm =
4952                         &crtc_state->wm.skl.optimal.planes[plane_id];
4953
4954                 if (wm->trans_wm.plane_res_b >= total[plane_id])
4955                         memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
4956         }
4957
4958         return 0;
4959 }
4960
4961 /*
4962  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4963  * for the read latency) and cpp should always be <= 8, so that
4964  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4965  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4966 */
4967 static uint_fixed_16_16_t
4968 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
4969                u8 cpp, u32 latency, u32 dbuf_block_size)
4970 {
4971         u32 wm_intermediate_val;
4972         uint_fixed_16_16_t ret;
4973
4974         if (latency == 0)
4975                 return FP_16_16_MAX;
4976
4977         wm_intermediate_val = latency * pixel_rate * cpp;
4978         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4979
4980         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4981                 ret = add_fixed16_u32(ret, 1);
4982
4983         return ret;
4984 }
4985
4986 static uint_fixed_16_16_t
4987 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
4988                uint_fixed_16_16_t plane_blocks_per_line)
4989 {
4990         u32 wm_intermediate_val;
4991         uint_fixed_16_16_t ret;
4992
4993         if (latency == 0)
4994                 return FP_16_16_MAX;
4995
4996         wm_intermediate_val = latency * pixel_rate;
4997         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4998                                            pipe_htotal * 1000);
4999         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5000         return ret;
5001 }
5002
5003 static uint_fixed_16_16_t
5004 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5005 {
5006         u32 pixel_rate;
5007         u32 crtc_htotal;
5008         uint_fixed_16_16_t linetime_us;
5009
5010         if (!crtc_state->hw.active)
5011                 return u32_to_fixed16(0);
5012
5013         pixel_rate = crtc_state->pixel_rate;
5014
5015         if (WARN_ON(pixel_rate == 0))
5016                 return u32_to_fixed16(0);
5017
5018         crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5019         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5020
5021         return linetime_us;
5022 }
5023
5024 static u32
5025 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5026                               const struct intel_plane_state *plane_state)
5027 {
5028         u64 adjusted_pixel_rate;
5029         uint_fixed_16_16_t downscale_amount;
5030
5031         /* Shouldn't reach here on disabled planes... */
5032         if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
5033                 return 0;
5034
5035         /*
5036          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5037          * with additional adjustments for plane-specific scaling.
5038          */
5039         adjusted_pixel_rate = crtc_state->pixel_rate;
5040         downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5041
5042         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5043                                             downscale_amount);
5044 }
5045
5046 static int
5047 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5048                       int width, const struct drm_format_info *format,
5049                       u64 modifier, unsigned int rotation,
5050                       u32 plane_pixel_rate, struct skl_wm_params *wp,
5051                       int color_plane)
5052 {
5053         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5054         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5055         u32 interm_pbpl;
5056
5057         /* only planar format has two planes */
5058         if (color_plane == 1 &&
5059             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5060                 drm_dbg_kms(&dev_priv->drm,
5061                             "Non planar format have single plane\n");
5062                 return -EINVAL;
5063         }
5064
5065         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5066                       modifier == I915_FORMAT_MOD_Yf_TILED ||
5067                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5068                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5069         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5070         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5071                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5072         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5073
5074         wp->width = width;
5075         if (color_plane == 1 && wp->is_planar)
5076                 wp->width /= 2;
5077
5078         wp->cpp = format->cpp[color_plane];
5079         wp->plane_pixel_rate = plane_pixel_rate;
5080
5081         if (INTEL_GEN(dev_priv) >= 11 &&
5082             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5083                 wp->dbuf_block_size = 256;
5084         else
5085                 wp->dbuf_block_size = 512;
5086
5087         if (drm_rotation_90_or_270(rotation)) {
5088                 switch (wp->cpp) {
5089                 case 1:
5090                         wp->y_min_scanlines = 16;
5091                         break;
5092                 case 2:
5093                         wp->y_min_scanlines = 8;
5094                         break;
5095                 case 4:
5096                         wp->y_min_scanlines = 4;
5097                         break;
5098                 default:
5099                         MISSING_CASE(wp->cpp);
5100                         return -EINVAL;
5101                 }
5102         } else {
5103                 wp->y_min_scanlines = 4;
5104         }
5105
5106         if (skl_needs_memory_bw_wa(dev_priv))
5107                 wp->y_min_scanlines *= 2;
5108
5109         wp->plane_bytes_per_line = wp->width * wp->cpp;
5110         if (wp->y_tiled) {
5111                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5112                                            wp->y_min_scanlines,
5113                                            wp->dbuf_block_size);
5114
5115                 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5116                         interm_pbpl++;
5117
5118                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5119                                                         wp->y_min_scanlines);
5120         } else {
5121                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5122                                            wp->dbuf_block_size);
5123
5124                 if (!wp->x_tiled ||
5125                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5126                         interm_pbpl++;
5127
5128                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5129         }
5130
5131         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5132                                              wp->plane_blocks_per_line);
5133
5134         wp->linetime_us = fixed16_to_u32_round_up(
5135                                         intel_get_linetime_us(crtc_state));
5136
5137         return 0;
5138 }
5139
5140 static int
5141 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5142                             const struct intel_plane_state *plane_state,
5143                             struct skl_wm_params *wp, int color_plane)
5144 {
5145         const struct drm_framebuffer *fb = plane_state->hw.fb;
5146         int width;
5147
5148         /*
5149          * Src coordinates are already rotated by 270 degrees for
5150          * the 90/270 degree plane rotation cases (to match the
5151          * GTT mapping), hence no need to account for rotation here.
5152          */
5153         width = drm_rect_width(&plane_state->uapi.src) >> 16;
5154
5155         return skl_compute_wm_params(crtc_state, width,
5156                                      fb->format, fb->modifier,
5157                                      plane_state->hw.rotation,
5158                                      skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5159                                      wp, color_plane);
5160 }
5161
5162 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5163 {
5164         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5165                 return true;
5166
5167         /* The number of lines are ignored for the level 0 watermark. */
5168         return level > 0;
5169 }
5170
5171 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5172                                  int level,
5173                                  unsigned int latency,
5174                                  const struct skl_wm_params *wp,
5175                                  const struct skl_wm_level *result_prev,
5176                                  struct skl_wm_level *result /* out */)
5177 {
5178         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5179         uint_fixed_16_16_t method1, method2;
5180         uint_fixed_16_16_t selected_result;
5181         u32 res_blocks, res_lines, min_ddb_alloc = 0;
5182
5183         if (latency == 0) {
5184                 /* reject it */
5185                 result->min_ddb_alloc = U16_MAX;
5186                 return;
5187         }
5188
5189         /*
5190          * WaIncreaseLatencyIPCEnabled: kbl,cfl
5191          * Display WA #1141: kbl,cfl
5192          */
5193         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
5194             dev_priv->ipc_enabled)
5195                 latency += 4;
5196
5197         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5198                 latency += 15;
5199
5200         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5201                                  wp->cpp, latency, wp->dbuf_block_size);
5202         method2 = skl_wm_method2(wp->plane_pixel_rate,
5203                                  crtc_state->hw.adjusted_mode.crtc_htotal,
5204                                  latency,
5205                                  wp->plane_blocks_per_line);
5206
5207         if (wp->y_tiled) {
5208                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5209         } else {
5210                 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5211                      wp->dbuf_block_size < 1) &&
5212                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5213                         selected_result = method2;
5214                 } else if (latency >= wp->linetime_us) {
5215                         if (IS_GEN(dev_priv, 9) &&
5216                             !IS_GEMINILAKE(dev_priv))
5217                                 selected_result = min_fixed16(method1, method2);
5218                         else
5219                                 selected_result = method2;
5220                 } else {
5221                         selected_result = method1;
5222                 }
5223         }
5224
5225         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5226         res_lines = div_round_up_fixed16(selected_result,
5227                                          wp->plane_blocks_per_line);
5228
5229         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5230                 /* Display WA #1125: skl,bxt,kbl */
5231                 if (level == 0 && wp->rc_surface)
5232                         res_blocks +=
5233                                 fixed16_to_u32_round_up(wp->y_tile_minimum);
5234
5235                 /* Display WA #1126: skl,bxt,kbl */
5236                 if (level >= 1 && level <= 7) {
5237                         if (wp->y_tiled) {
5238                                 res_blocks +=
5239                                     fixed16_to_u32_round_up(wp->y_tile_minimum);
5240                                 res_lines += wp->y_min_scanlines;
5241                         } else {
5242                                 res_blocks++;
5243                         }
5244
5245                         /*
5246                          * Make sure result blocks for higher latency levels are
5247                          * atleast as high as level below the current level.
5248                          * Assumption in DDB algorithm optimization for special
5249                          * cases. Also covers Display WA #1125 for RC.
5250                          */
5251                         if (result_prev->plane_res_b > res_blocks)
5252                                 res_blocks = result_prev->plane_res_b;
5253                 }
5254         }
5255
5256         if (INTEL_GEN(dev_priv) >= 11) {
5257                 if (wp->y_tiled) {
5258                         int extra_lines;
5259
5260                         if (res_lines % wp->y_min_scanlines == 0)
5261                                 extra_lines = wp->y_min_scanlines;
5262                         else
5263                                 extra_lines = wp->y_min_scanlines * 2 -
5264                                         res_lines % wp->y_min_scanlines;
5265
5266                         min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5267                                                                  wp->plane_blocks_per_line);
5268                 } else {
5269                         min_ddb_alloc = res_blocks +
5270                                 DIV_ROUND_UP(res_blocks, 10);
5271                 }
5272         }
5273
5274         if (!skl_wm_has_lines(dev_priv, level))
5275                 res_lines = 0;
5276
5277         if (res_lines > 31) {
5278                 /* reject it */
5279                 result->min_ddb_alloc = U16_MAX;
5280                 return;
5281         }
5282
5283         /*
5284          * If res_lines is valid, assume we can use this watermark level
5285          * for now.  We'll come back and disable it after we calculate the
5286          * DDB allocation if it turns out we don't actually have enough
5287          * blocks to satisfy it.
5288          */
5289         result->plane_res_b = res_blocks;
5290         result->plane_res_l = res_lines;
5291         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5292         result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5293         result->plane_en = true;
5294 }
5295
5296 static void
5297 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5298                       const struct skl_wm_params *wm_params,
5299                       struct skl_wm_level *levels)
5300 {
5301         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5302         int level, max_level = ilk_wm_max_level(dev_priv);
5303         struct skl_wm_level *result_prev = &levels[0];
5304
5305         for (level = 0; level <= max_level; level++) {
5306                 struct skl_wm_level *result = &levels[level];
5307                 unsigned int latency = dev_priv->wm.skl_latency[level];
5308
5309                 skl_compute_plane_wm(crtc_state, level, latency,
5310                                      wm_params, result_prev, result);
5311
5312                 result_prev = result;
5313         }
5314 }
5315
5316 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5317                                 const struct skl_wm_params *wm_params,
5318                                 struct skl_plane_wm *plane_wm)
5319 {
5320         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5321         struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5322         struct skl_wm_level *levels = plane_wm->wm;
5323         unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5324
5325         skl_compute_plane_wm(crtc_state, 0, latency,
5326                              wm_params, &levels[0],
5327                              sagv_wm);
5328 }
5329
5330 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5331                                       const struct skl_wm_params *wp,
5332                                       struct skl_plane_wm *wm)
5333 {
5334         struct drm_device *dev = crtc_state->uapi.crtc->dev;
5335         const struct drm_i915_private *dev_priv = to_i915(dev);
5336         u16 trans_min, trans_amount, trans_y_tile_min;
5337         u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5338
5339         /* Transition WM don't make any sense if ipc is disabled */
5340         if (!dev_priv->ipc_enabled)
5341                 return;
5342
5343         /*
5344          * WaDisableTWM:skl,kbl,cfl,bxt
5345          * Transition WM are not recommended by HW team for GEN9
5346          */
5347         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5348                 return;
5349
5350         if (INTEL_GEN(dev_priv) >= 11)
5351                 trans_min = 4;
5352         else
5353                 trans_min = 14;
5354
5355         /* Display WA #1140: glk,cnl */
5356         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5357                 trans_amount = 0;
5358         else
5359                 trans_amount = 10; /* This is configurable amount */
5360
5361         trans_offset_b = trans_min + trans_amount;
5362
5363         /*
5364          * The spec asks for Selected Result Blocks for wm0 (the real value),
5365          * not Result Blocks (the integer value). Pay attention to the capital
5366          * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5367          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5368          * and since we later will have to get the ceiling of the sum in the
5369          * transition watermarks calculation, we can just pretend Selected
5370          * Result Blocks is Result Blocks minus 1 and it should work for the
5371          * current platforms.
5372          */
5373         wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5374
5375         if (wp->y_tiled) {
5376                 trans_y_tile_min =
5377                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5378                 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5379                                 trans_offset_b;
5380         } else {
5381                 res_blocks = wm0_sel_res_b + trans_offset_b;
5382         }
5383
5384         /*
5385          * Just assume we can enable the transition watermark.  After
5386          * computing the DDB we'll come back and disable it if that
5387          * assumption turns out to be false.
5388          */
5389         wm->trans_wm.plane_res_b = res_blocks + 1;
5390         wm->trans_wm.plane_en = true;
5391 }
5392
5393 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5394                                      const struct intel_plane_state *plane_state,
5395                                      enum plane_id plane_id, int color_plane)
5396 {
5397         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5398         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5399         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5400         struct skl_wm_params wm_params;
5401         int ret;
5402
5403         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5404                                           &wm_params, color_plane);
5405         if (ret)
5406                 return ret;
5407
5408         skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5409
5410         if (INTEL_GEN(dev_priv) >= 12)
5411                 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5412
5413         skl_compute_transition_wm(crtc_state, &wm_params, wm);
5414
5415         return 0;
5416 }
5417
5418 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5419                                  const struct intel_plane_state *plane_state,
5420                                  enum plane_id plane_id)
5421 {
5422         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5423         struct skl_wm_params wm_params;
5424         int ret;
5425
5426         wm->is_planar = true;
5427
5428         /* uv plane watermarks must also be validated for NV12/Planar */
5429         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5430                                           &wm_params, 1);
5431         if (ret)
5432                 return ret;
5433
5434         skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5435
5436         return 0;
5437 }
5438
5439 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5440                               const struct intel_plane_state *plane_state)
5441 {
5442         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5443         const struct drm_framebuffer *fb = plane_state->hw.fb;
5444         enum plane_id plane_id = plane->id;
5445         int ret;
5446
5447         if (!intel_wm_plane_visible(crtc_state, plane_state))
5448                 return 0;
5449
5450         ret = skl_build_plane_wm_single(crtc_state, plane_state,
5451                                         plane_id, 0);
5452         if (ret)
5453                 return ret;
5454
5455         if (fb->format->is_yuv && fb->format->num_planes > 1) {
5456                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5457                                             plane_id);
5458                 if (ret)
5459                         return ret;
5460         }
5461
5462         return 0;
5463 }
5464
5465 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5466                               const struct intel_plane_state *plane_state)
5467 {
5468         enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5469         int ret;
5470
5471         /* Watermarks calculated in master */
5472         if (plane_state->planar_slave)
5473                 return 0;
5474
5475         if (plane_state->planar_linked_plane) {
5476                 const struct drm_framebuffer *fb = plane_state->hw.fb;
5477                 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5478
5479                 WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state));
5480                 WARN_ON(!fb->format->is_yuv ||
5481                         fb->format->num_planes == 1);
5482
5483                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5484                                                 y_plane_id, 0);
5485                 if (ret)
5486                         return ret;
5487
5488                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5489                                                 plane_id, 1);
5490                 if (ret)
5491                         return ret;
5492         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5493                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5494                                                 plane_id, 0);
5495                 if (ret)
5496                         return ret;
5497         }
5498
5499         return 0;
5500 }
5501
5502 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5503 {
5504         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5505         struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5506         struct intel_plane *plane;
5507         const struct intel_plane_state *plane_state;
5508         int ret;
5509
5510         /*
5511          * We'll only calculate watermarks for planes that are actually
5512          * enabled, so make sure all other planes are set as disabled.
5513          */
5514         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5515
5516         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5517                                                      crtc_state) {
5518
5519                 if (INTEL_GEN(dev_priv) >= 11)
5520                         ret = icl_build_plane_wm(crtc_state, plane_state);
5521                 else
5522                         ret = skl_build_plane_wm(crtc_state, plane_state);
5523                 if (ret)
5524                         return ret;
5525         }
5526
5527         return 0;
5528 }
5529
5530 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5531                                 i915_reg_t reg,
5532                                 const struct skl_ddb_entry *entry)
5533 {
5534         if (entry->end)
5535                 intel_de_write_fw(dev_priv, reg,
5536                                   (entry->end - 1) << 16 | entry->start);
5537         else
5538                 intel_de_write_fw(dev_priv, reg, 0);
5539 }
5540
5541 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5542                                i915_reg_t reg,
5543                                const struct skl_wm_level *level)
5544 {
5545         u32 val = 0;
5546
5547         if (level->plane_en)
5548                 val |= PLANE_WM_EN;
5549         if (level->ignore_lines)
5550                 val |= PLANE_WM_IGNORE_LINES;
5551         val |= level->plane_res_b;
5552         val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5553
5554         intel_de_write_fw(dev_priv, reg, val);
5555 }
5556
5557 void skl_write_plane_wm(struct intel_plane *plane,
5558                         const struct intel_crtc_state *crtc_state)
5559 {
5560         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5561         int level, max_level = ilk_wm_max_level(dev_priv);
5562         enum plane_id plane_id = plane->id;
5563         enum pipe pipe = plane->pipe;
5564         const struct skl_plane_wm *wm =
5565                 &crtc_state->wm.skl.optimal.planes[plane_id];
5566         const struct skl_ddb_entry *ddb_y =
5567                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5568         const struct skl_ddb_entry *ddb_uv =
5569                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5570
5571         for (level = 0; level <= max_level; level++) {
5572                 const struct skl_wm_level *wm_level;
5573
5574                 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5575
5576                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5577                                    wm_level);
5578         }
5579         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5580                            &wm->trans_wm);
5581
5582         if (INTEL_GEN(dev_priv) >= 11) {
5583                 skl_ddb_entry_write(dev_priv,
5584                                     PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5585                 return;
5586         }
5587
5588         if (wm->is_planar)
5589                 swap(ddb_y, ddb_uv);
5590
5591         skl_ddb_entry_write(dev_priv,
5592                             PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5593         skl_ddb_entry_write(dev_priv,
5594                             PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5595 }
5596
5597 void skl_write_cursor_wm(struct intel_plane *plane,
5598                          const struct intel_crtc_state *crtc_state)
5599 {
5600         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5601         int level, max_level = ilk_wm_max_level(dev_priv);
5602         enum plane_id plane_id = plane->id;
5603         enum pipe pipe = plane->pipe;
5604         const struct skl_plane_wm *wm =
5605                 &crtc_state->wm.skl.optimal.planes[plane_id];
5606         const struct skl_ddb_entry *ddb =
5607                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5608
5609         for (level = 0; level <= max_level; level++) {
5610                 const struct skl_wm_level *wm_level;
5611
5612                 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5613
5614                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5615                                    wm_level);
5616         }
5617         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5618
5619         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5620 }
5621
5622 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5623                          const struct skl_wm_level *l2)
5624 {
5625         return l1->plane_en == l2->plane_en &&
5626                 l1->ignore_lines == l2->ignore_lines &&
5627                 l1->plane_res_l == l2->plane_res_l &&
5628                 l1->plane_res_b == l2->plane_res_b;
5629 }
5630
5631 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5632                                 const struct skl_plane_wm *wm1,
5633                                 const struct skl_plane_wm *wm2)
5634 {
5635         int level, max_level = ilk_wm_max_level(dev_priv);
5636
5637         for (level = 0; level <= max_level; level++) {
5638                 /*
5639                  * We don't check uv_wm as the hardware doesn't actually
5640                  * use it. It only gets used for calculating the required
5641                  * ddb allocation.
5642                  */
5643                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5644                         return false;
5645         }
5646
5647         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5648 }
5649
5650 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5651                                     const struct skl_ddb_entry *b)
5652 {
5653         return a->start < b->end && b->start < a->end;
5654 }
5655
5656 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5657                                  const struct skl_ddb_entry *entries,
5658                                  int num_entries, int ignore_idx)
5659 {
5660         int i;
5661
5662         for (i = 0; i < num_entries; i++) {
5663                 if (i != ignore_idx &&
5664                     skl_ddb_entries_overlap(ddb, &entries[i]))
5665                         return true;
5666         }
5667
5668         return false;
5669 }
5670
5671 static int
5672 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5673                             struct intel_crtc_state *new_crtc_state)
5674 {
5675         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5676         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5677         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5678         struct intel_plane *plane;
5679
5680         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5681                 struct intel_plane_state *plane_state;
5682                 enum plane_id plane_id = plane->id;
5683
5684                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5685                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5686                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5687                                         &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5688                         continue;
5689
5690                 plane_state = intel_atomic_get_plane_state(state, plane);
5691                 if (IS_ERR(plane_state))
5692                         return PTR_ERR(plane_state);
5693
5694                 new_crtc_state->update_planes |= BIT(plane_id);
5695         }
5696
5697         return 0;
5698 }
5699
5700 static int
5701 skl_compute_ddb(struct intel_atomic_state *state)
5702 {
5703         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5704         struct intel_crtc_state *old_crtc_state;
5705         struct intel_crtc_state *new_crtc_state;
5706         struct intel_crtc *crtc;
5707         int ret, i;
5708
5709         state->enabled_dbuf_slices_mask = dev_priv->enabled_dbuf_slices_mask;
5710
5711         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5712                                             new_crtc_state, i) {
5713                 ret = skl_allocate_pipe_ddb(new_crtc_state);
5714                 if (ret)
5715                         return ret;
5716
5717                 ret = skl_ddb_add_affected_planes(old_crtc_state,
5718                                                   new_crtc_state);
5719                 if (ret)
5720                         return ret;
5721         }
5722
5723         return 0;
5724 }
5725
5726 static char enast(bool enable)
5727 {
5728         return enable ? '*' : ' ';
5729 }
5730
5731 static void
5732 skl_print_wm_changes(struct intel_atomic_state *state)
5733 {
5734         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5735         const struct intel_crtc_state *old_crtc_state;
5736         const struct intel_crtc_state *new_crtc_state;
5737         struct intel_plane *plane;
5738         struct intel_crtc *crtc;
5739         int i;
5740
5741         if (!drm_debug_enabled(DRM_UT_KMS))
5742                 return;
5743
5744         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5745                                             new_crtc_state, i) {
5746                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5747
5748                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5749                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5750
5751                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5752                         enum plane_id plane_id = plane->id;
5753                         const struct skl_ddb_entry *old, *new;
5754
5755                         old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5756                         new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5757
5758                         if (skl_ddb_entry_equal(old, new))
5759                                 continue;
5760
5761                         drm_dbg_kms(&dev_priv->drm,
5762                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5763                                     plane->base.base.id, plane->base.name,
5764                                     old->start, old->end, new->start, new->end,
5765                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5766                 }
5767
5768                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5769                         enum plane_id plane_id = plane->id;
5770                         const struct skl_plane_wm *old_wm, *new_wm;
5771
5772                         old_wm = &old_pipe_wm->planes[plane_id];
5773                         new_wm = &new_pipe_wm->planes[plane_id];
5774
5775                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5776                                 continue;
5777
5778                         drm_dbg_kms(&dev_priv->drm,
5779                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5780                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5781                                     plane->base.base.id, plane->base.name,
5782                                     enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5783                                     enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5784                                     enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5785                                     enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5786                                     enast(old_wm->trans_wm.plane_en),
5787                                     enast(old_wm->sagv_wm0.plane_en),
5788                                     enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5789                                     enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5790                                     enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5791                                     enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5792                                     enast(new_wm->trans_wm.plane_en),
5793                                     enast(new_wm->sagv_wm0.plane_en));
5794
5795                         drm_dbg_kms(&dev_priv->drm,
5796                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5797                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5798                                     plane->base.base.id, plane->base.name,
5799                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5800                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5801                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5802                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5803                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5804                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5805                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5806                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5807                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5808                                     enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5809
5810                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5811                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5812                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5813                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5814                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5815                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5816                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5817                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5818                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5819                                     enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5820
5821                         drm_dbg_kms(&dev_priv->drm,
5822                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5823                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5824                                     plane->base.base.id, plane->base.name,
5825                                     old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5826                                     old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5827                                     old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5828                                     old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5829                                     old_wm->trans_wm.plane_res_b,
5830                                     old_wm->sagv_wm0.plane_res_b,
5831                                     new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5832                                     new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5833                                     new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5834                                     new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5835                                     new_wm->trans_wm.plane_res_b,
5836                                     new_wm->sagv_wm0.plane_res_b);
5837
5838                         drm_dbg_kms(&dev_priv->drm,
5839                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5840                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5841                                     plane->base.base.id, plane->base.name,
5842                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5843                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5844                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5845                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5846                                     old_wm->trans_wm.min_ddb_alloc,
5847                                     old_wm->sagv_wm0.min_ddb_alloc,
5848                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5849                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5850                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5851                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5852                                     new_wm->trans_wm.min_ddb_alloc,
5853                                     new_wm->sagv_wm0.min_ddb_alloc);
5854                 }
5855         }
5856 }
5857
5858 static int intel_add_all_pipes(struct intel_atomic_state *state)
5859 {
5860         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5861         struct intel_crtc *crtc;
5862
5863         for_each_intel_crtc(&dev_priv->drm, crtc) {
5864                 struct intel_crtc_state *crtc_state;
5865
5866                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5867                 if (IS_ERR(crtc_state))
5868                         return PTR_ERR(crtc_state);
5869         }
5870
5871         return 0;
5872 }
5873
5874 static int
5875 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5876 {
5877         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5878         int ret;
5879
5880         /*
5881          * If this is our first atomic update following hardware readout,
5882          * we can't trust the DDB that the BIOS programmed for us.  Let's
5883          * pretend that all pipes switched active status so that we'll
5884          * ensure a full DDB recompute.
5885          */
5886         if (dev_priv->wm.distrust_bios_wm) {
5887                 ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
5888                                        state->base.acquire_ctx);
5889                 if (ret)
5890                         return ret;
5891
5892                 state->active_pipe_changes = INTEL_INFO(dev_priv)->pipe_mask;
5893
5894                 /*
5895                  * We usually only initialize state->active_pipes if we
5896                  * we're doing a modeset; make sure this field is always
5897                  * initialized during the sanitization process that happens
5898                  * on the first commit too.
5899                  */
5900                 if (!state->modeset)
5901                         state->active_pipes = dev_priv->active_pipes;
5902         }
5903
5904         /*
5905          * If the modeset changes which CRTC's are active, we need to
5906          * recompute the DDB allocation for *all* active pipes, even
5907          * those that weren't otherwise being modified in any way by this
5908          * atomic commit.  Due to the shrinking of the per-pipe allocations
5909          * when new active CRTC's are added, it's possible for a pipe that
5910          * we were already using and aren't changing at all here to suddenly
5911          * become invalid if its DDB needs exceeds its new allocation.
5912          *
5913          * Note that if we wind up doing a full DDB recompute, we can't let
5914          * any other display updates race with this transaction, so we need
5915          * to grab the lock on *all* CRTC's.
5916          */
5917         if (state->active_pipe_changes || state->modeset) {
5918                 ret = intel_add_all_pipes(state);
5919                 if (ret)
5920                         return ret;
5921         }
5922
5923         return 0;
5924 }
5925
5926 /*
5927  * To make sure the cursor watermark registers are always consistent
5928  * with our computed state the following scenario needs special
5929  * treatment:
5930  *
5931  * 1. enable cursor
5932  * 2. move cursor entirely offscreen
5933  * 3. disable cursor
5934  *
5935  * Step 2. does call .disable_plane() but does not zero the watermarks
5936  * (since we consider an offscreen cursor still active for the purposes
5937  * of watermarks). Step 3. would not normally call .disable_plane()
5938  * because the actual plane visibility isn't changing, and we don't
5939  * deallocate the cursor ddb until the pipe gets disabled. So we must
5940  * force step 3. to call .disable_plane() to update the watermark
5941  * registers properly.
5942  *
5943  * Other planes do not suffer from this issues as their watermarks are
5944  * calculated based on the actual plane visibility. The only time this
5945  * can trigger for the other planes is during the initial readout as the
5946  * default value of the watermarks registers is not zero.
5947  */
5948 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
5949                                       struct intel_crtc *crtc)
5950 {
5951         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5952         const struct intel_crtc_state *old_crtc_state =
5953                 intel_atomic_get_old_crtc_state(state, crtc);
5954         struct intel_crtc_state *new_crtc_state =
5955                 intel_atomic_get_new_crtc_state(state, crtc);
5956         struct intel_plane *plane;
5957
5958         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5959                 struct intel_plane_state *plane_state;
5960                 enum plane_id plane_id = plane->id;
5961
5962                 /*
5963                  * Force a full wm update for every plane on modeset.
5964                  * Required because the reset value of the wm registers
5965                  * is non-zero, whereas we want all disabled planes to
5966                  * have zero watermarks. So if we turn off the relevant
5967                  * power well the hardware state will go out of sync
5968                  * with the software state.
5969                  */
5970                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
5971                     skl_plane_wm_equals(dev_priv,
5972                                         &old_crtc_state->wm.skl.optimal.planes[plane_id],
5973                                         &new_crtc_state->wm.skl.optimal.planes[plane_id]))
5974                         continue;
5975
5976                 plane_state = intel_atomic_get_plane_state(state, plane);
5977                 if (IS_ERR(plane_state))
5978                         return PTR_ERR(plane_state);
5979
5980                 new_crtc_state->update_planes |= BIT(plane_id);
5981         }
5982
5983         return 0;
5984 }
5985
5986 static int
5987 skl_compute_wm(struct intel_atomic_state *state)
5988 {
5989         struct intel_crtc *crtc;
5990         struct intel_crtc_state *new_crtc_state;
5991         struct intel_crtc_state *old_crtc_state;
5992         int ret, i;
5993
5994         ret = skl_ddb_add_affected_pipes(state);
5995         if (ret)
5996                 return ret;
5997
5998         /*
5999          * Calculate WM's for all pipes that are part of this transaction.
6000          * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6001          * weren't otherwise being modified if pipe allocations had to change.
6002          */
6003         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6004                                             new_crtc_state, i) {
6005                 ret = skl_build_pipe_wm(new_crtc_state);
6006                 if (ret)
6007                         return ret;
6008         }
6009
6010         ret = skl_compute_ddb(state);
6011         if (ret)
6012                 return ret;
6013
6014         ret = intel_compute_sagv_mask(state);
6015         if (ret)
6016                 return ret;
6017
6018         /*
6019          * skl_compute_ddb() will have adjusted the final watermarks
6020          * based on how much ddb is available. Now we can actually
6021          * check if the final watermarks changed.
6022          */
6023         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6024                                             new_crtc_state, i) {
6025                 ret = skl_wm_add_affected_planes(state, crtc);
6026                 if (ret)
6027                         return ret;
6028         }
6029
6030         skl_print_wm_changes(state);
6031
6032         return 0;
6033 }
6034
6035 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6036                                   struct intel_wm_config *config)
6037 {
6038         struct intel_crtc *crtc;
6039
6040         /* Compute the currently _active_ config */
6041         for_each_intel_crtc(&dev_priv->drm, crtc) {
6042                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6043
6044                 if (!wm->pipe_enabled)
6045                         continue;
6046
6047                 config->sprites_enabled |= wm->sprites_enabled;
6048                 config->sprites_scaled |= wm->sprites_scaled;
6049                 config->num_pipes_active++;
6050         }
6051 }
6052
6053 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6054 {
6055         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6056         struct ilk_wm_maximums max;
6057         struct intel_wm_config config = {};
6058         struct ilk_wm_values results = {};
6059         enum intel_ddb_partitioning partitioning;
6060
6061         ilk_compute_wm_config(dev_priv, &config);
6062
6063         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6064         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6065
6066         /* 5/6 split only in single pipe config on IVB+ */
6067         if (INTEL_GEN(dev_priv) >= 7 &&
6068             config.num_pipes_active == 1 && config.sprites_enabled) {
6069                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6070                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6071
6072                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6073         } else {
6074                 best_lp_wm = &lp_wm_1_2;
6075         }
6076
6077         partitioning = (best_lp_wm == &lp_wm_1_2) ?
6078                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6079
6080         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6081
6082         ilk_write_wm_values(dev_priv, &results);
6083 }
6084
6085 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6086                                    struct intel_crtc *crtc)
6087 {
6088         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6089         const struct intel_crtc_state *crtc_state =
6090                 intel_atomic_get_new_crtc_state(state, crtc);
6091
6092         mutex_lock(&dev_priv->wm.wm_mutex);
6093         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6094         ilk_program_watermarks(dev_priv);
6095         mutex_unlock(&dev_priv->wm.wm_mutex);
6096 }
6097
6098 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6099                                     struct intel_crtc *crtc)
6100 {
6101         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6102         const struct intel_crtc_state *crtc_state =
6103                 intel_atomic_get_new_crtc_state(state, crtc);
6104
6105         if (!crtc_state->wm.need_postvbl_update)
6106                 return;
6107
6108         mutex_lock(&dev_priv->wm.wm_mutex);
6109         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6110         ilk_program_watermarks(dev_priv);
6111         mutex_unlock(&dev_priv->wm.wm_mutex);
6112 }
6113
6114 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6115 {
6116         level->plane_en = val & PLANE_WM_EN;
6117         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6118         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6119         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6120                 PLANE_WM_LINES_MASK;
6121 }
6122
6123 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6124                               struct skl_pipe_wm *out)
6125 {
6126         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6127         enum pipe pipe = crtc->pipe;
6128         int level, max_level;
6129         enum plane_id plane_id;
6130         u32 val;
6131
6132         max_level = ilk_wm_max_level(dev_priv);
6133
6134         for_each_plane_id_on_crtc(crtc, plane_id) {
6135                 struct skl_plane_wm *wm = &out->planes[plane_id];
6136
6137                 for (level = 0; level <= max_level; level++) {
6138                         if (plane_id != PLANE_CURSOR)
6139                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6140                         else
6141                                 val = I915_READ(CUR_WM(pipe, level));
6142
6143                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
6144                 }
6145
6146                 if (INTEL_GEN(dev_priv) >= 12)
6147                         wm->sagv_wm0 = wm->wm[0];
6148
6149                 if (plane_id != PLANE_CURSOR)
6150                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6151                 else
6152                         val = I915_READ(CUR_WM_TRANS(pipe));
6153
6154                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6155         }
6156
6157         if (!crtc->active)
6158                 return;
6159 }
6160
6161 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6162 {
6163         struct intel_crtc *crtc;
6164         struct intel_crtc_state *crtc_state;
6165
6166         skl_ddb_get_hw_state(dev_priv);
6167         for_each_intel_crtc(&dev_priv->drm, crtc) {
6168                 crtc_state = to_intel_crtc_state(crtc->base.state);
6169
6170                 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6171         }
6172
6173         if (dev_priv->active_pipes) {
6174                 /* Fully recompute DDB on first atomic commit */
6175                 dev_priv->wm.distrust_bios_wm = true;
6176         }
6177 }
6178
6179 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6180 {
6181         struct drm_device *dev = crtc->base.dev;
6182         struct drm_i915_private *dev_priv = to_i915(dev);
6183         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6184         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6185         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6186         enum pipe pipe = crtc->pipe;
6187         static const i915_reg_t wm0_pipe_reg[] = {
6188                 [PIPE_A] = WM0_PIPEA_ILK,
6189                 [PIPE_B] = WM0_PIPEB_ILK,
6190                 [PIPE_C] = WM0_PIPEC_IVB,
6191         };
6192
6193         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6194
6195         memset(active, 0, sizeof(*active));
6196
6197         active->pipe_enabled = crtc->active;
6198
6199         if (active->pipe_enabled) {
6200                 u32 tmp = hw->wm_pipe[pipe];
6201
6202                 /*
6203                  * For active pipes LP0 watermark is marked as
6204                  * enabled, and LP1+ watermaks as disabled since
6205                  * we can't really reverse compute them in case
6206                  * multiple pipes are active.
6207                  */
6208                 active->wm[0].enable = true;
6209                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6210                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6211                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6212         } else {
6213                 int level, max_level = ilk_wm_max_level(dev_priv);
6214
6215                 /*
6216                  * For inactive pipes, all watermark levels
6217                  * should be marked as enabled but zeroed,
6218                  * which is what we'd compute them to.
6219                  */
6220                 for (level = 0; level <= max_level; level++)
6221                         active->wm[level].enable = true;
6222         }
6223
6224         crtc->wm.active.ilk = *active;
6225 }
6226
6227 #define _FW_WM(value, plane) \
6228         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6229 #define _FW_WM_VLV(value, plane) \
6230         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6231
6232 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6233                                struct g4x_wm_values *wm)
6234 {
6235         u32 tmp;
6236
6237         tmp = I915_READ(DSPFW1);
6238         wm->sr.plane = _FW_WM(tmp, SR);
6239         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6240         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6241         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6242
6243         tmp = I915_READ(DSPFW2);
6244         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6245         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6246         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6247         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6248         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6249         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6250
6251         tmp = I915_READ(DSPFW3);
6252         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6253         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6254         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6255         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6256 }
6257
6258 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6259                                struct vlv_wm_values *wm)
6260 {
6261         enum pipe pipe;
6262         u32 tmp;
6263
6264         for_each_pipe(dev_priv, pipe) {
6265                 tmp = I915_READ(VLV_DDL(pipe));
6266
6267                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6268                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6269                 wm->ddl[pipe].plane[PLANE_CURSOR] =
6270                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6271                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6272                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6273                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6274                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6275         }
6276
6277         tmp = I915_READ(DSPFW1);
6278         wm->sr.plane = _FW_WM(tmp, SR);
6279         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6280         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6281         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6282
6283         tmp = I915_READ(DSPFW2);
6284         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6285         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6286         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6287
6288         tmp = I915_READ(DSPFW3);
6289         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6290
6291         if (IS_CHERRYVIEW(dev_priv)) {
6292                 tmp = I915_READ(DSPFW7_CHV);
6293                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6294                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6295
6296                 tmp = I915_READ(DSPFW8_CHV);
6297                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6298                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6299
6300                 tmp = I915_READ(DSPFW9_CHV);
6301                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6302                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6303
6304                 tmp = I915_READ(DSPHOWM);
6305                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6306                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6307                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6308                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6309                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6310                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6311                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6312                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6313                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6314                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6315         } else {
6316                 tmp = I915_READ(DSPFW7);
6317                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6318                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6319
6320                 tmp = I915_READ(DSPHOWM);
6321                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6322                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6323                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6324                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6325                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6326                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6327                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6328         }
6329 }
6330
6331 #undef _FW_WM
6332 #undef _FW_WM_VLV
6333
6334 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6335 {
6336         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6337         struct intel_crtc *crtc;
6338
6339         g4x_read_wm_values(dev_priv, wm);
6340
6341         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6342
6343         for_each_intel_crtc(&dev_priv->drm, crtc) {
6344                 struct intel_crtc_state *crtc_state =
6345                         to_intel_crtc_state(crtc->base.state);
6346                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6347                 struct g4x_pipe_wm *raw;
6348                 enum pipe pipe = crtc->pipe;
6349                 enum plane_id plane_id;
6350                 int level, max_level;
6351
6352                 active->cxsr = wm->cxsr;
6353                 active->hpll_en = wm->hpll_en;
6354                 active->fbc_en = wm->fbc_en;
6355
6356                 active->sr = wm->sr;
6357                 active->hpll = wm->hpll;
6358
6359                 for_each_plane_id_on_crtc(crtc, plane_id) {
6360                         active->wm.plane[plane_id] =
6361                                 wm->pipe[pipe].plane[plane_id];
6362                 }
6363
6364                 if (wm->cxsr && wm->hpll_en)
6365                         max_level = G4X_WM_LEVEL_HPLL;
6366                 else if (wm->cxsr)
6367                         max_level = G4X_WM_LEVEL_SR;
6368                 else
6369                         max_level = G4X_WM_LEVEL_NORMAL;
6370
6371                 level = G4X_WM_LEVEL_NORMAL;
6372                 raw = &crtc_state->wm.g4x.raw[level];
6373                 for_each_plane_id_on_crtc(crtc, plane_id)
6374                         raw->plane[plane_id] = active->wm.plane[plane_id];
6375
6376                 if (++level > max_level)
6377                         goto out;
6378
6379                 raw = &crtc_state->wm.g4x.raw[level];
6380                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6381                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6382                 raw->plane[PLANE_SPRITE0] = 0;
6383                 raw->fbc = active->sr.fbc;
6384
6385                 if (++level > max_level)
6386                         goto out;
6387
6388                 raw = &crtc_state->wm.g4x.raw[level];
6389                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6390                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6391                 raw->plane[PLANE_SPRITE0] = 0;
6392                 raw->fbc = active->hpll.fbc;
6393
6394         out:
6395                 for_each_plane_id_on_crtc(crtc, plane_id)
6396                         g4x_raw_plane_wm_set(crtc_state, level,
6397                                              plane_id, USHRT_MAX);
6398                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6399
6400                 crtc_state->wm.g4x.optimal = *active;
6401                 crtc_state->wm.g4x.intermediate = *active;
6402
6403                 drm_dbg_kms(&dev_priv->drm,
6404                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6405                             pipe_name(pipe),
6406                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6407                             wm->pipe[pipe].plane[PLANE_CURSOR],
6408                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
6409         }
6410
6411         drm_dbg_kms(&dev_priv->drm,
6412                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6413                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6414         drm_dbg_kms(&dev_priv->drm,
6415                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6416                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6417         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6418                     yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6419 }
6420
6421 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6422 {
6423         struct intel_plane *plane;
6424         struct intel_crtc *crtc;
6425
6426         mutex_lock(&dev_priv->wm.wm_mutex);
6427
6428         for_each_intel_plane(&dev_priv->drm, plane) {
6429                 struct intel_crtc *crtc =
6430                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6431                 struct intel_crtc_state *crtc_state =
6432                         to_intel_crtc_state(crtc->base.state);
6433                 struct intel_plane_state *plane_state =
6434                         to_intel_plane_state(plane->base.state);
6435                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6436                 enum plane_id plane_id = plane->id;
6437                 int level;
6438
6439                 if (plane_state->uapi.visible)
6440                         continue;
6441
6442                 for (level = 0; level < 3; level++) {
6443                         struct g4x_pipe_wm *raw =
6444                                 &crtc_state->wm.g4x.raw[level];
6445
6446                         raw->plane[plane_id] = 0;
6447                         wm_state->wm.plane[plane_id] = 0;
6448                 }
6449
6450                 if (plane_id == PLANE_PRIMARY) {
6451                         for (level = 0; level < 3; level++) {
6452                                 struct g4x_pipe_wm *raw =
6453                                         &crtc_state->wm.g4x.raw[level];
6454                                 raw->fbc = 0;
6455                         }
6456
6457                         wm_state->sr.fbc = 0;
6458                         wm_state->hpll.fbc = 0;
6459                         wm_state->fbc_en = false;
6460                 }
6461         }
6462
6463         for_each_intel_crtc(&dev_priv->drm, crtc) {
6464                 struct intel_crtc_state *crtc_state =
6465                         to_intel_crtc_state(crtc->base.state);
6466
6467                 crtc_state->wm.g4x.intermediate =
6468                         crtc_state->wm.g4x.optimal;
6469                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6470         }
6471
6472         g4x_program_watermarks(dev_priv);
6473
6474         mutex_unlock(&dev_priv->wm.wm_mutex);
6475 }
6476
6477 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6478 {
6479         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6480         struct intel_crtc *crtc;
6481         u32 val;
6482
6483         vlv_read_wm_values(dev_priv, wm);
6484
6485         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6486         wm->level = VLV_WM_LEVEL_PM2;
6487
6488         if (IS_CHERRYVIEW(dev_priv)) {
6489                 vlv_punit_get(dev_priv);
6490
6491                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6492                 if (val & DSP_MAXFIFO_PM5_ENABLE)
6493                         wm->level = VLV_WM_LEVEL_PM5;
6494
6495                 /*
6496                  * If DDR DVFS is disabled in the BIOS, Punit
6497                  * will never ack the request. So if that happens
6498                  * assume we don't have to enable/disable DDR DVFS
6499                  * dynamically. To test that just set the REQ_ACK
6500                  * bit to poke the Punit, but don't change the
6501                  * HIGH/LOW bits so that we don't actually change
6502                  * the current state.
6503                  */
6504                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6505                 val |= FORCE_DDR_FREQ_REQ_ACK;
6506                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6507
6508                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6509                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6510                         drm_dbg_kms(&dev_priv->drm,
6511                                     "Punit not acking DDR DVFS request, "
6512                                     "assuming DDR DVFS is disabled\n");
6513                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6514                 } else {
6515                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6516                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6517                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6518                 }
6519
6520                 vlv_punit_put(dev_priv);
6521         }
6522
6523         for_each_intel_crtc(&dev_priv->drm, crtc) {
6524                 struct intel_crtc_state *crtc_state =
6525                         to_intel_crtc_state(crtc->base.state);
6526                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6527                 const struct vlv_fifo_state *fifo_state =
6528                         &crtc_state->wm.vlv.fifo_state;
6529                 enum pipe pipe = crtc->pipe;
6530                 enum plane_id plane_id;
6531                 int level;
6532
6533                 vlv_get_fifo_size(crtc_state);
6534
6535                 active->num_levels = wm->level + 1;
6536                 active->cxsr = wm->cxsr;
6537
6538                 for (level = 0; level < active->num_levels; level++) {
6539                         struct g4x_pipe_wm *raw =
6540                                 &crtc_state->wm.vlv.raw[level];
6541
6542                         active->sr[level].plane = wm->sr.plane;
6543                         active->sr[level].cursor = wm->sr.cursor;
6544
6545                         for_each_plane_id_on_crtc(crtc, plane_id) {
6546                                 active->wm[level].plane[plane_id] =
6547                                         wm->pipe[pipe].plane[plane_id];
6548
6549                                 raw->plane[plane_id] =
6550                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
6551                                                             fifo_state->plane[plane_id]);
6552                         }
6553                 }
6554
6555                 for_each_plane_id_on_crtc(crtc, plane_id)
6556                         vlv_raw_plane_wm_set(crtc_state, level,
6557                                              plane_id, USHRT_MAX);
6558                 vlv_invalidate_wms(crtc, active, level);
6559
6560                 crtc_state->wm.vlv.optimal = *active;
6561                 crtc_state->wm.vlv.intermediate = *active;
6562
6563                 drm_dbg_kms(&dev_priv->drm,
6564                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6565                             pipe_name(pipe),
6566                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6567                             wm->pipe[pipe].plane[PLANE_CURSOR],
6568                             wm->pipe[pipe].plane[PLANE_SPRITE0],
6569                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
6570         }
6571
6572         drm_dbg_kms(&dev_priv->drm,
6573                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6574                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6575 }
6576
6577 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6578 {
6579         struct intel_plane *plane;
6580         struct intel_crtc *crtc;
6581
6582         mutex_lock(&dev_priv->wm.wm_mutex);
6583
6584         for_each_intel_plane(&dev_priv->drm, plane) {
6585                 struct intel_crtc *crtc =
6586                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6587                 struct intel_crtc_state *crtc_state =
6588                         to_intel_crtc_state(crtc->base.state);
6589                 struct intel_plane_state *plane_state =
6590                         to_intel_plane_state(plane->base.state);
6591                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6592                 const struct vlv_fifo_state *fifo_state =
6593                         &crtc_state->wm.vlv.fifo_state;
6594                 enum plane_id plane_id = plane->id;
6595                 int level;
6596
6597                 if (plane_state->uapi.visible)
6598                         continue;
6599
6600                 for (level = 0; level < wm_state->num_levels; level++) {
6601                         struct g4x_pipe_wm *raw =
6602                                 &crtc_state->wm.vlv.raw[level];
6603
6604                         raw->plane[plane_id] = 0;
6605
6606                         wm_state->wm[level].plane[plane_id] =
6607                                 vlv_invert_wm_value(raw->plane[plane_id],
6608                                                     fifo_state->plane[plane_id]);
6609                 }
6610         }
6611
6612         for_each_intel_crtc(&dev_priv->drm, crtc) {
6613                 struct intel_crtc_state *crtc_state =
6614                         to_intel_crtc_state(crtc->base.state);
6615
6616                 crtc_state->wm.vlv.intermediate =
6617                         crtc_state->wm.vlv.optimal;
6618                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6619         }
6620
6621         vlv_program_watermarks(dev_priv);
6622
6623         mutex_unlock(&dev_priv->wm.wm_mutex);
6624 }
6625
6626 /*
6627  * FIXME should probably kill this and improve
6628  * the real watermark readout/sanitation instead
6629  */
6630 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6631 {
6632         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6633         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6634         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6635
6636         /*
6637          * Don't touch WM1S_LP_EN here.
6638          * Doing so could cause underruns.
6639          */
6640 }
6641
6642 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6643 {
6644         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6645         struct intel_crtc *crtc;
6646
6647         ilk_init_lp_watermarks(dev_priv);
6648
6649         for_each_intel_crtc(&dev_priv->drm, crtc)
6650                 ilk_pipe_wm_get_hw_state(crtc);
6651
6652         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6653         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6654         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6655
6656         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6657         if (INTEL_GEN(dev_priv) >= 7) {
6658                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6659                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6660         }
6661
6662         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6663                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6664                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6665         else if (IS_IVYBRIDGE(dev_priv))
6666                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6667                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6668
6669         hw->enable_fbc_wm =
6670                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6671 }
6672
6673 /**
6674  * intel_update_watermarks - update FIFO watermark values based on current modes
6675  * @crtc: the #intel_crtc on which to compute the WM
6676  *
6677  * Calculate watermark values for the various WM regs based on current mode
6678  * and plane configuration.
6679  *
6680  * There are several cases to deal with here:
6681  *   - normal (i.e. non-self-refresh)
6682  *   - self-refresh (SR) mode
6683  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6684  *   - lines are small relative to FIFO size (buffer can hold more than 2
6685  *     lines), so need to account for TLB latency
6686  *
6687  *   The normal calculation is:
6688  *     watermark = dotclock * bytes per pixel * latency
6689  *   where latency is platform & configuration dependent (we assume pessimal
6690  *   values here).
6691  *
6692  *   The SR calculation is:
6693  *     watermark = (trunc(latency/line time)+1) * surface width *
6694  *       bytes per pixel
6695  *   where
6696  *     line time = htotal / dotclock
6697  *     surface width = hdisplay for normal plane and 64 for cursor
6698  *   and latency is assumed to be high, as above.
6699  *
6700  * The final value programmed to the register should always be rounded up,
6701  * and include an extra 2 entries to account for clock crossings.
6702  *
6703  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6704  * to set the non-SR watermarks to 8.
6705  */
6706 void intel_update_watermarks(struct intel_crtc *crtc)
6707 {
6708         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6709
6710         if (dev_priv->display.update_wm)
6711                 dev_priv->display.update_wm(crtc);
6712 }
6713
6714 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6715 {
6716         u32 val;
6717
6718         if (!HAS_IPC(dev_priv))
6719                 return;
6720
6721         val = I915_READ(DISP_ARB_CTL2);
6722
6723         if (dev_priv->ipc_enabled)
6724                 val |= DISP_IPC_ENABLE;
6725         else
6726                 val &= ~DISP_IPC_ENABLE;
6727
6728         I915_WRITE(DISP_ARB_CTL2, val);
6729 }
6730
6731 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6732 {
6733         /* Display WA #0477 WaDisableIPC: skl */
6734         if (IS_SKYLAKE(dev_priv))
6735                 return false;
6736
6737         /* Display WA #1141: SKL:all KBL:all CFL */
6738         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
6739                 return dev_priv->dram_info.symmetric_memory;
6740
6741         return true;
6742 }
6743
6744 void intel_init_ipc(struct drm_i915_private *dev_priv)
6745 {
6746         if (!HAS_IPC(dev_priv))
6747                 return;
6748
6749         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6750
6751         intel_enable_ipc(dev_priv);
6752 }
6753
6754 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6755 {
6756         /*
6757          * On Ibex Peak and Cougar Point, we need to disable clock
6758          * gating for the panel power sequencer or it will fail to
6759          * start up when no ports are active.
6760          */
6761         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6762 }
6763
6764 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6765 {
6766         enum pipe pipe;
6767
6768         for_each_pipe(dev_priv, pipe) {
6769                 I915_WRITE(DSPCNTR(pipe),
6770                            I915_READ(DSPCNTR(pipe)) |
6771                            DISPPLANE_TRICKLE_FEED_DISABLE);
6772
6773                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6774                 POSTING_READ(DSPSURF(pipe));
6775         }
6776 }
6777
6778 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6779 {
6780         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6781
6782         /*
6783          * Required for FBC
6784          * WaFbcDisableDpfcClockGating:ilk
6785          */
6786         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6787                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6788                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6789
6790         I915_WRITE(PCH_3DCGDIS0,
6791                    MARIUNIT_CLOCK_GATE_DISABLE |
6792                    SVSMUNIT_CLOCK_GATE_DISABLE);
6793         I915_WRITE(PCH_3DCGDIS1,
6794                    VFMUNIT_CLOCK_GATE_DISABLE);
6795
6796         /*
6797          * According to the spec the following bits should be set in
6798          * order to enable memory self-refresh
6799          * The bit 22/21 of 0x42004
6800          * The bit 5 of 0x42020
6801          * The bit 15 of 0x45000
6802          */
6803         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6804                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
6805                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6806         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6807         I915_WRITE(DISP_ARB_CTL,
6808                    (I915_READ(DISP_ARB_CTL) |
6809                     DISP_FBC_WM_DIS));
6810
6811         /*
6812          * Based on the document from hardware guys the following bits
6813          * should be set unconditionally in order to enable FBC.
6814          * The bit 22 of 0x42000
6815          * The bit 22 of 0x42004
6816          * The bit 7,8,9 of 0x42020.
6817          */
6818         if (IS_IRONLAKE_M(dev_priv)) {
6819                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6820                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6821                            I915_READ(ILK_DISPLAY_CHICKEN1) |
6822                            ILK_FBCQ_DIS);
6823                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6824                            I915_READ(ILK_DISPLAY_CHICKEN2) |
6825                            ILK_DPARB_GATE);
6826         }
6827
6828         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6829
6830         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6831                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6832                    ILK_ELPIN_409_SELECT);
6833
6834         g4x_disable_trickle_feed(dev_priv);
6835
6836         ibx_init_clock_gating(dev_priv);
6837 }
6838
6839 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6840 {
6841         enum pipe pipe;
6842         u32 val;
6843
6844         /*
6845          * On Ibex Peak and Cougar Point, we need to disable clock
6846          * gating for the panel power sequencer or it will fail to
6847          * start up when no ports are active.
6848          */
6849         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6850                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6851                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6852         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6853                    DPLS_EDP_PPS_FIX_DIS);
6854         /* The below fixes the weird display corruption, a few pixels shifted
6855          * downward, on (only) LVDS of some HP laptops with IVY.
6856          */
6857         for_each_pipe(dev_priv, pipe) {
6858                 val = I915_READ(TRANS_CHICKEN2(pipe));
6859                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6860                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6861                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6862                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6863                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6864                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6865                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6866         }
6867         /* WADP0ClockGatingDisable */
6868         for_each_pipe(dev_priv, pipe) {
6869                 I915_WRITE(TRANS_CHICKEN1(pipe),
6870                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6871         }
6872 }
6873
6874 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6875 {
6876         u32 tmp;
6877
6878         tmp = I915_READ(MCH_SSKPD);
6879         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6880                 drm_dbg_kms(&dev_priv->drm,
6881                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6882                             tmp);
6883 }
6884
6885 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6886 {
6887         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6888
6889         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6890
6891         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6892                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6893                    ILK_ELPIN_409_SELECT);
6894
6895         I915_WRITE(GEN6_UCGCTL1,
6896                    I915_READ(GEN6_UCGCTL1) |
6897                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6898                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6899
6900         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
6901          * gating disable must be set.  Failure to set it results in
6902          * flickering pixels due to Z write ordering failures after
6903          * some amount of runtime in the Mesa "fire" demo, and Unigine
6904          * Sanctuary and Tropics, and apparently anything else with
6905          * alpha test or pixel discard.
6906          *
6907          * According to the spec, bit 11 (RCCUNIT) must also be set,
6908          * but we didn't debug actual testcases to find it out.
6909          *
6910          * WaDisableRCCUnitClockGating:snb
6911          * WaDisableRCPBUnitClockGating:snb
6912          */
6913         I915_WRITE(GEN6_UCGCTL2,
6914                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
6915                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
6916
6917         /*
6918          * According to the spec the following bits should be
6919          * set in order to enable memory self-refresh and fbc:
6920          * The bit21 and bit22 of 0x42000
6921          * The bit21 and bit22 of 0x42004
6922          * The bit5 and bit7 of 0x42020
6923          * The bit14 of 0x70180
6924          * The bit14 of 0x71180
6925          *
6926          * WaFbcAsynchFlipDisableFbcQueue:snb
6927          */
6928         I915_WRITE(ILK_DISPLAY_CHICKEN1,
6929                    I915_READ(ILK_DISPLAY_CHICKEN1) |
6930                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
6931         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6932                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6933                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
6934         I915_WRITE(ILK_DSPCLK_GATE_D,
6935                    I915_READ(ILK_DSPCLK_GATE_D) |
6936                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
6937                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6938
6939         g4x_disable_trickle_feed(dev_priv);
6940
6941         cpt_init_clock_gating(dev_priv);
6942
6943         gen6_check_mch_setup(dev_priv);
6944 }
6945
6946 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
6947 {
6948         /*
6949          * TODO: this bit should only be enabled when really needed, then
6950          * disabled when not needed anymore in order to save power.
6951          */
6952         if (HAS_PCH_LPT_LP(dev_priv))
6953                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
6954                            I915_READ(SOUTH_DSPCLK_GATE_D) |
6955                            PCH_LP_PARTITION_LEVEL_DISABLE);
6956
6957         /* WADPOClockGatingDisable:hsw */
6958         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
6959                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
6960                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6961 }
6962
6963 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
6964 {
6965         if (HAS_PCH_LPT_LP(dev_priv)) {
6966                 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
6967
6968                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
6969                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
6970         }
6971 }
6972
6973 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
6974                                    int general_prio_credits,
6975                                    int high_prio_credits)
6976 {
6977         u32 misccpctl;
6978         u32 val;
6979
6980         /* WaTempDisableDOPClkGating:bdw */
6981         misccpctl = I915_READ(GEN7_MISCCPCTL);
6982         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
6983
6984         val = I915_READ(GEN8_L3SQCREG1);
6985         val &= ~L3_PRIO_CREDITS_MASK;
6986         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
6987         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
6988         I915_WRITE(GEN8_L3SQCREG1, val);
6989
6990         /*
6991          * Wait at least 100 clocks before re-enabling clock gating.
6992          * See the definition of L3SQCREG1 in BSpec.
6993          */
6994         POSTING_READ(GEN8_L3SQCREG1);
6995         udelay(1);
6996         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
6997 }
6998
6999 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7000 {
7001         /* This is not an Wa. Enable to reduce Sampler power */
7002         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7003                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7004
7005         /*Wa_14010594013:icl, ehl */
7006         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7007                          0, CNL_DELAY_PMRSP);
7008 }
7009
7010 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7011 {
7012         u32 vd_pg_enable = 0;
7013         unsigned int i;
7014
7015         /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7016         for (i = 0; i < I915_MAX_VCS; i++) {
7017                 if (HAS_ENGINE(dev_priv, _VCS(i)))
7018                         vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7019                                         VDN_MFX_POWERGATE_ENABLE(i);
7020         }
7021
7022         I915_WRITE(POWERGATE_ENABLE,
7023                    I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7024
7025         /* Wa_1409825376:tgl (pre-prod)*/
7026         if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
7027                 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7028                            TGL_VRH_GATING_DIS);
7029
7030         /* Wa_14011059788:tgl */
7031         intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7032                          0, DFR_DISABLE);
7033 }
7034
7035 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7036 {
7037         if (!HAS_PCH_CNP(dev_priv))
7038                 return;
7039
7040         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7041         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7042                    CNP_PWM_CGE_GATING_DISABLE);
7043 }
7044
7045 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7046 {
7047         u32 val;
7048         cnp_init_clock_gating(dev_priv);
7049
7050         /* This is not an Wa. Enable for better image quality */
7051         I915_WRITE(_3D_CHICKEN3,
7052                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7053
7054         /* WaEnableChickenDCPR:cnl */
7055         I915_WRITE(GEN8_CHICKEN_DCPR_1,
7056                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7057
7058         /* WaFbcWakeMemOn:cnl */
7059         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7060                    DISP_FBC_MEMORY_WAKE);
7061
7062         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7063         /* ReadHitWriteOnlyDisable:cnl */
7064         val |= RCCUNIT_CLKGATE_DIS;
7065         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7066
7067         /* Wa_2201832410:cnl */
7068         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7069         val |= GWUNIT_CLKGATE_DIS;
7070         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7071
7072         /* WaDisableVFclkgate:cnl */
7073         /* WaVFUnitClockGatingDisable:cnl */
7074         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7075         val |= VFUNIT_CLKGATE_DIS;
7076         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7077 }
7078
7079 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7080 {
7081         cnp_init_clock_gating(dev_priv);
7082         gen9_init_clock_gating(dev_priv);
7083
7084         /* WaFbcNukeOnHostModify:cfl */
7085         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7086                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7087 }
7088
7089 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7090 {
7091         gen9_init_clock_gating(dev_priv);
7092
7093         /* WaDisableSDEUnitClockGating:kbl */
7094         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7095                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7096                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7097
7098         /* WaDisableGamClockGating:kbl */
7099         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7100                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7101                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7102
7103         /* WaFbcNukeOnHostModify:kbl */
7104         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7105                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7106 }
7107
7108 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7109 {
7110         gen9_init_clock_gating(dev_priv);
7111
7112         /* WAC6entrylatency:skl */
7113         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7114                    FBC_LLC_FULLY_OPEN);
7115
7116         /* WaFbcNukeOnHostModify:skl */
7117         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7118                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7119 }
7120
7121 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7122 {
7123         enum pipe pipe;
7124
7125         /* WaSwitchSolVfFArbitrationPriority:bdw */
7126         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7127
7128         /* WaPsrDPAMaskVBlankInSRD:bdw */
7129         I915_WRITE(CHICKEN_PAR1_1,
7130                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7131
7132         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7133         for_each_pipe(dev_priv, pipe) {
7134                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7135                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
7136                            BDW_DPRS_MASK_VBLANK_SRD);
7137         }
7138
7139         /* WaVSRefCountFullforceMissDisable:bdw */
7140         /* WaDSRefCountFullforceMissDisable:bdw */
7141         I915_WRITE(GEN7_FF_THREAD_MODE,
7142                    I915_READ(GEN7_FF_THREAD_MODE) &
7143                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7144
7145         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7146                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7147
7148         /* WaDisableSDEUnitClockGating:bdw */
7149         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7150                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7151
7152         /* WaProgramL3SqcReg1Default:bdw */
7153         gen8_set_l3sqc_credits(dev_priv, 30, 2);
7154
7155         /* WaKVMNotificationOnConfigChange:bdw */
7156         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7157                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7158
7159         lpt_init_clock_gating(dev_priv);
7160
7161         /* WaDisableDopClockGating:bdw
7162          *
7163          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7164          * clock gating.
7165          */
7166         I915_WRITE(GEN6_UCGCTL1,
7167                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7168 }
7169
7170 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7171 {
7172         /* This is required by WaCatErrorRejectionIssue:hsw */
7173         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7174                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7175                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7176
7177         /* WaSwitchSolVfFArbitrationPriority:hsw */
7178         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7179
7180         lpt_init_clock_gating(dev_priv);
7181 }
7182
7183 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7184 {
7185         u32 snpcr;
7186
7187         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7188
7189         /* WaDisableBackToBackFlipFix:ivb */
7190         I915_WRITE(IVB_CHICKEN3,
7191                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7192                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7193
7194         if (IS_IVB_GT1(dev_priv))
7195                 I915_WRITE(GEN7_ROW_CHICKEN2,
7196                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7197         else {
7198                 /* must write both registers */
7199                 I915_WRITE(GEN7_ROW_CHICKEN2,
7200                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7201                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7202                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7203         }
7204
7205         /*
7206          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7207          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7208          */
7209         I915_WRITE(GEN6_UCGCTL2,
7210                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7211
7212         /* This is required by WaCatErrorRejectionIssue:ivb */
7213         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7214                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7215                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7216
7217         g4x_disable_trickle_feed(dev_priv);
7218
7219         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7220         snpcr &= ~GEN6_MBC_SNPCR_MASK;
7221         snpcr |= GEN6_MBC_SNPCR_MED;
7222         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7223
7224         if (!HAS_PCH_NOP(dev_priv))
7225                 cpt_init_clock_gating(dev_priv);
7226
7227         gen6_check_mch_setup(dev_priv);
7228 }
7229
7230 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7231 {
7232         /* WaDisableBackToBackFlipFix:vlv */
7233         I915_WRITE(IVB_CHICKEN3,
7234                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7235                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7236
7237         /* WaDisableDopClockGating:vlv */
7238         I915_WRITE(GEN7_ROW_CHICKEN2,
7239                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7240
7241         /* This is required by WaCatErrorRejectionIssue:vlv */
7242         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7243                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7244                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7245
7246         /*
7247          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7248          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7249          */
7250         I915_WRITE(GEN6_UCGCTL2,
7251                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7252
7253         /* WaDisableL3Bank2xClockGate:vlv
7254          * Disabling L3 clock gating- MMIO 940c[25] = 1
7255          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7256         I915_WRITE(GEN7_UCGCTL4,
7257                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7258
7259         /*
7260          * WaDisableVLVClockGating_VBIIssue:vlv
7261          * Disable clock gating on th GCFG unit to prevent a delay
7262          * in the reporting of vblank events.
7263          */
7264         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7265 }
7266
7267 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7268 {
7269         /* WaVSRefCountFullforceMissDisable:chv */
7270         /* WaDSRefCountFullforceMissDisable:chv */
7271         I915_WRITE(GEN7_FF_THREAD_MODE,
7272                    I915_READ(GEN7_FF_THREAD_MODE) &
7273                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7274
7275         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7276         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7277                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7278
7279         /* WaDisableCSUnitClockGating:chv */
7280         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7281                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7282
7283         /* WaDisableSDEUnitClockGating:chv */
7284         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7285                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7286
7287         /*
7288          * WaProgramL3SqcReg1Default:chv
7289          * See gfxspecs/Related Documents/Performance Guide/
7290          * LSQC Setting Recommendations.
7291          */
7292         gen8_set_l3sqc_credits(dev_priv, 38, 2);
7293 }
7294
7295 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7296 {
7297         u32 dspclk_gate;
7298
7299         I915_WRITE(RENCLK_GATE_D1, 0);
7300         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7301                    GS_UNIT_CLOCK_GATE_DISABLE |
7302                    CL_UNIT_CLOCK_GATE_DISABLE);
7303         I915_WRITE(RAMCLK_GATE_D, 0);
7304         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7305                 OVRUNIT_CLOCK_GATE_DISABLE |
7306                 OVCUNIT_CLOCK_GATE_DISABLE;
7307         if (IS_GM45(dev_priv))
7308                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7309         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7310
7311         g4x_disable_trickle_feed(dev_priv);
7312 }
7313
7314 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7315 {
7316         struct intel_uncore *uncore = &dev_priv->uncore;
7317
7318         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7319         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7320         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7321         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7322         intel_uncore_write16(uncore, DEUC, 0);
7323         intel_uncore_write(uncore,
7324                            MI_ARB_STATE,
7325                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7326 }
7327
7328 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7329 {
7330         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7331                    I965_RCC_CLOCK_GATE_DISABLE |
7332                    I965_RCPB_CLOCK_GATE_DISABLE |
7333                    I965_ISC_CLOCK_GATE_DISABLE |
7334                    I965_FBC_CLOCK_GATE_DISABLE);
7335         I915_WRITE(RENCLK_GATE_D2, 0);
7336         I915_WRITE(MI_ARB_STATE,
7337                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7338 }
7339
7340 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7341 {
7342         u32 dstate = I915_READ(D_STATE);
7343
7344         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7345                 DSTATE_DOT_CLOCK_GATING;
7346         I915_WRITE(D_STATE, dstate);
7347
7348         if (IS_PINEVIEW(dev_priv))
7349                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7350
7351         /* IIR "flip pending" means done if this bit is set */
7352         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7353
7354         /* interrupts should cause a wake up from C3 */
7355         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7356
7357         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7358         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7359
7360         I915_WRITE(MI_ARB_STATE,
7361                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7362 }
7363
7364 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7365 {
7366         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7367
7368         /* interrupts should cause a wake up from C3 */
7369         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7370                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7371
7372         I915_WRITE(MEM_MODE,
7373                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7374 }
7375
7376 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7377 {
7378         I915_WRITE(MEM_MODE,
7379                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7380                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7381 }
7382
7383 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7384 {
7385         dev_priv->display.init_clock_gating(dev_priv);
7386 }
7387
7388 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7389 {
7390         if (HAS_PCH_LPT(dev_priv))
7391                 lpt_suspend_hw(dev_priv);
7392 }
7393
7394 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7395 {
7396         drm_dbg_kms(&dev_priv->drm,
7397                     "No clock gating settings or workarounds applied.\n");
7398 }
7399
7400 /**
7401  * intel_init_clock_gating_hooks - setup the clock gating hooks
7402  * @dev_priv: device private
7403  *
7404  * Setup the hooks that configure which clocks of a given platform can be
7405  * gated and also apply various GT and display specific workarounds for these
7406  * platforms. Note that some GT specific workarounds are applied separately
7407  * when GPU contexts or batchbuffers start their execution.
7408  */
7409 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7410 {
7411         if (IS_GEN(dev_priv, 12))
7412                 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7413         else if (IS_GEN(dev_priv, 11))
7414                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7415         else if (IS_CANNONLAKE(dev_priv))
7416                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7417         else if (IS_COFFEELAKE(dev_priv))
7418                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7419         else if (IS_SKYLAKE(dev_priv))
7420                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7421         else if (IS_KABYLAKE(dev_priv))
7422                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7423         else if (IS_BROXTON(dev_priv))
7424                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7425         else if (IS_GEMINILAKE(dev_priv))
7426                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7427         else if (IS_BROADWELL(dev_priv))
7428                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7429         else if (IS_CHERRYVIEW(dev_priv))
7430                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7431         else if (IS_HASWELL(dev_priv))
7432                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7433         else if (IS_IVYBRIDGE(dev_priv))
7434                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7435         else if (IS_VALLEYVIEW(dev_priv))
7436                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7437         else if (IS_GEN(dev_priv, 6))
7438                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7439         else if (IS_GEN(dev_priv, 5))
7440                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7441         else if (IS_G4X(dev_priv))
7442                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7443         else if (IS_I965GM(dev_priv))
7444                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7445         else if (IS_I965G(dev_priv))
7446                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7447         else if (IS_GEN(dev_priv, 3))
7448                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7449         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7450                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7451         else if (IS_GEN(dev_priv, 2))
7452                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7453         else {
7454                 MISSING_CASE(INTEL_DEVID(dev_priv));
7455                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7456         }
7457 }
7458
7459 /* Set up chip specific power management-related functions */
7460 void intel_init_pm(struct drm_i915_private *dev_priv)
7461 {
7462         /* For cxsr */
7463         if (IS_PINEVIEW(dev_priv))
7464                 pnv_get_mem_freq(dev_priv);
7465         else if (IS_GEN(dev_priv, 5))
7466                 ilk_get_mem_freq(dev_priv);
7467
7468         if (intel_has_sagv(dev_priv))
7469                 skl_setup_sagv_block_time(dev_priv);
7470
7471         /* For FIFO watermark updates */
7472         if (INTEL_GEN(dev_priv) >= 9) {
7473                 skl_setup_wm_latency(dev_priv);
7474                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7475         } else if (HAS_PCH_SPLIT(dev_priv)) {
7476                 ilk_setup_wm_latency(dev_priv);
7477
7478                 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7479                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7480                     (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7481                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7482                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7483                         dev_priv->display.compute_intermediate_wm =
7484                                 ilk_compute_intermediate_wm;
7485                         dev_priv->display.initial_watermarks =
7486                                 ilk_initial_watermarks;
7487                         dev_priv->display.optimize_watermarks =
7488                                 ilk_optimize_watermarks;
7489                 } else {
7490                         drm_dbg_kms(&dev_priv->drm,
7491                                     "Failed to read display plane latency. "
7492                                     "Disable CxSR\n");
7493                 }
7494         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7495                 vlv_setup_wm_latency(dev_priv);
7496                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7497                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7498                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7499                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7500                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7501         } else if (IS_G4X(dev_priv)) {
7502                 g4x_setup_wm_latency(dev_priv);
7503                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7504                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7505                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7506                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7507         } else if (IS_PINEVIEW(dev_priv)) {
7508                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7509                                             dev_priv->is_ddr3,
7510                                             dev_priv->fsb_freq,
7511                                             dev_priv->mem_freq)) {
7512                         drm_info(&dev_priv->drm,
7513                                  "failed to find known CxSR latency "
7514                                  "(found ddr%s fsb freq %d, mem freq %d), "
7515                                  "disabling CxSR\n",
7516                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7517                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7518                         /* Disable CxSR and never update its watermark again */
7519                         intel_set_memory_cxsr(dev_priv, false);
7520                         dev_priv->display.update_wm = NULL;
7521                 } else
7522                         dev_priv->display.update_wm = pnv_update_wm;
7523         } else if (IS_GEN(dev_priv, 4)) {
7524                 dev_priv->display.update_wm = i965_update_wm;
7525         } else if (IS_GEN(dev_priv, 3)) {
7526                 dev_priv->display.update_wm = i9xx_update_wm;
7527                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7528         } else if (IS_GEN(dev_priv, 2)) {
7529                 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7530                         dev_priv->display.update_wm = i845_update_wm;
7531                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7532                 } else {
7533                         dev_priv->display.update_wm = i9xx_update_wm;
7534                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7535                 }
7536         } else {
7537                 drm_err(&dev_priv->drm,
7538                         "unexpected fall-through in %s\n", __func__);
7539         }
7540 }
7541
7542 void intel_pm_setup(struct drm_i915_private *dev_priv)
7543 {
7544         dev_priv->runtime_pm.suspended = false;
7545         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7546 }