drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include <drm/drm_atomic_helper.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include "display/intel_atomic.h"
  36 #include "display/intel_bw.h"
  37 #include "display/intel_display_types.h"
  38 #include "display/intel_fbc.h"
  39 #include "display/intel_sprite.h"
  40
  41 #include "gt/intel_llc.h"
  42
  43 #include "i915_drv.h"
  44 #include "i915_fixed.h"
  45 #include "i915_irq.h"
  46 #include "i915_trace.h"
  47 #include "intel_pm.h"
  48 #include "intel_sideband.h"
  49 #include "../../../platform/x86/intel_ips.h"
  50
  51 /* Stores plane specific WM parameters */
  52 struct skl_wm_params {
  53         bool x_tiled, y_tiled;
  54         bool rc_surface;
  55         bool is_planar;
  56         u32 width;
  57         u8 cpp;
  58         u32 plane_pixel_rate;
  59         u32 y_min_scanlines;
  60         u32 plane_bytes_per_line;
  61         uint_fixed_16_16_t plane_blocks_per_line;
  62         uint_fixed_16_16_t y_tile_minimum;
  63         u32 linetime_us;
  64         u32 dbuf_block_size;
  65 };
  66
  67 /* used in computing the new watermarks state */
  68 struct intel_wm_config {
  69         unsigned int num_pipes_active;
  70         bool sprites_enabled;
  71         bool sprites_scaled;
  72 };
  73
  74 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  75 {
  76         if (HAS_LLC(dev_priv)) {
  77                 /*
  78                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  79                  * Display WA #0390: skl,kbl
  80                  *
  81                  * Must match Sampler, Pixel Back End, and Media. See
  82                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  83                  */
  84                 I915_WRITE(CHICKEN_PAR1_1,
  85                            I915_READ(CHICKEN_PAR1_1) |
  86                            SKL_DE_COMPRESSED_HASH_MODE);
  87         }
  88
  89         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  90         I915_WRITE(CHICKEN_PAR1_1,
  91                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  92
  93         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  94         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  95                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  96
  97         /*
  98          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  99          * Display WA #0859: skl,bxt,kbl,glk,cfl
 100          */
 101         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 102                    DISP_FBC_MEMORY_WAKE);
 103
 104         if (IS_SKYLAKE(dev_priv)) {
 105                 /* WaDisableDopClockGating */
 106                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
 107                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 108         }
 109 }
 110
 111 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 112 {
 113         gen9_init_clock_gating(dev_priv);
 114
 115         /* WaDisableSDEUnitClockGating:bxt */
 116         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 117                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 118
 119         /*
 120          * FIXME:
 121          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 122          */
 123         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 124                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 125
 126         /*
 127          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 128          * to stay fully on.
 129          */
 130         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 131                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 132
 133         /*
 134          * Lower the display internal timeout.
 135          * This is needed to avoid any hard hangs when DSI port PLL
 136          * is off and a MMIO access is attempted by any privilege
 137          * application, using batch buffers or any other means.
 138          */
 139         I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 140
 141         /*
 142          * WaFbcTurnOffFbcWatermark:bxt
 143          * Display WA #0562: bxt
 144          */
 145         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 146                    DISP_FBC_WM_DIS);
 147
 148         /*
 149          * WaFbcHighMemBwCorruptionAvoidance:bxt
 150          * Display WA #0883: bxt
 151          */
 152         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
 153                    ILK_DPFC_DISABLE_DUMMY0);
 154 }
 155
 156 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 157 {
 158         gen9_init_clock_gating(dev_priv);
 159
 160         /*
 161          * WaDisablePWMClockGating:glk
 162          * Backlight PWM may stop in the asserted state, causing backlight
 163          * to stay fully on.
 164          */
 165         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 166                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 167 }
 168
 169 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 170 {
 171         u32 tmp;
 172
 173         tmp = I915_READ(CLKCFG);
 174
 175         switch (tmp & CLKCFG_FSB_MASK) {
 176         case CLKCFG_FSB_533:
 177                 dev_priv->fsb_freq = 533; /* 133*4 */
 178                 break;
 179         case CLKCFG_FSB_800:
 180                 dev_priv->fsb_freq = 800; /* 200*4 */
 181                 break;
 182         case CLKCFG_FSB_667:
 183                 dev_priv->fsb_freq =  667; /* 167*4 */
 184                 break;
 185         case CLKCFG_FSB_400:
 186                 dev_priv->fsb_freq = 400; /* 100*4 */
 187                 break;
 188         }
 189
 190         switch (tmp & CLKCFG_MEM_MASK) {
 191         case CLKCFG_MEM_533:
 192                 dev_priv->mem_freq = 533;
 193                 break;
 194         case CLKCFG_MEM_667:
 195                 dev_priv->mem_freq = 667;
 196                 break;
 197         case CLKCFG_MEM_800:
 198                 dev_priv->mem_freq = 800;
 199                 break;
 200         }
 201
 202         /* detect pineview DDR3 setting */
 203         tmp = I915_READ(CSHRDDR3CTL);
 204         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 205 }
 206
 207 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 208 {
 209         u16 ddrpll, csipll;
 210
 211         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 212         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 213
 214         switch (ddrpll & 0xff) {
 215         case 0xc:
 216                 dev_priv->mem_freq = 800;
 217                 break;
 218         case 0x10:
 219                 dev_priv->mem_freq = 1066;
 220                 break;
 221         case 0x14:
 222                 dev_priv->mem_freq = 1333;
 223                 break;
 224         case 0x18:
 225                 dev_priv->mem_freq = 1600;
 226                 break;
 227         default:
 228                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 229                         ddrpll & 0xff);
 230                 dev_priv->mem_freq = 0;
 231                 break;
 232         }
 233
 234         switch (csipll & 0x3ff) {
 235         case 0x00c:
 236                 dev_priv->fsb_freq = 3200;
 237                 break;
 238         case 0x00e:
 239                 dev_priv->fsb_freq = 3733;
 240                 break;
 241         case 0x010:
 242                 dev_priv->fsb_freq = 4266;
 243                 break;
 244         case 0x012:
 245                 dev_priv->fsb_freq = 4800;
 246                 break;
 247         case 0x014:
 248                 dev_priv->fsb_freq = 5333;
 249                 break;
 250         case 0x016:
 251                 dev_priv->fsb_freq = 5866;
 252                 break;
 253         case 0x018:
 254                 dev_priv->fsb_freq = 6400;
 255                 break;
 256         default:
 257                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 258                         csipll & 0x3ff);
 259                 dev_priv->fsb_freq = 0;
 260                 break;
 261         }
 262 }
 263
 264 static const struct cxsr_latency cxsr_latency_table[] = {
 265         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 266         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 267         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 268         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 269         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 270
 271         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 272         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 273         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 274         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 275         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 276
 277         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 278         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 279         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 280         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 281         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 282
 283         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 284         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 285         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 286         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 287         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 288
 289         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 290         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 291         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 292         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 293         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 294
 295         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 296         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 297         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 298         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 299         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 300 };
 301
 302 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 303                                                          bool is_ddr3,
 304                                                          int fsb,
 305                                                          int mem)
 306 {
 307         const struct cxsr_latency *latency;
 308         int i;
 309
 310         if (fsb == 0 || mem == 0)
 311                 return NULL;
 312
 313         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 314                 latency = &cxsr_latency_table[i];
 315                 if (is_desktop == latency->is_desktop &&
 316                     is_ddr3 == latency->is_ddr3 &&
 317                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 318                         return latency;
 319         }
 320
 321         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 322
 323         return NULL;
 324 }
 325
 326 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 327 {
 328         u32 val;
 329
 330         vlv_punit_get(dev_priv);
 331
 332         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 333         if (enable)
 334                 val &= ~FORCE_DDR_HIGH_FREQ;
 335         else
 336                 val |= FORCE_DDR_HIGH_FREQ;
 337         val &= ~FORCE_DDR_LOW_FREQ;
 338         val |= FORCE_DDR_FREQ_REQ_ACK;
 339         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 340
 341         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 342                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 343                 drm_err(&dev_priv->drm,
 344                         "timed out waiting for Punit DDR DVFS request\n");
 345
 346         vlv_punit_put(dev_priv);
 347 }
 348
 349 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 350 {
 351         u32 val;
 352
 353         vlv_punit_get(dev_priv);
 354
 355         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 356         if (enable)
 357                 val |= DSP_MAXFIFO_PM5_ENABLE;
 358         else
 359                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 360         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 361
 362         vlv_punit_put(dev_priv);
 363 }
 364
 365 #define FW_WM(value, plane) \
 366         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 367
 368 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 369 {
 370         bool was_enabled;
 371         u32 val;
 372
 373         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 374                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 375                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 376                 POSTING_READ(FW_BLC_SELF_VLV);
 377         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 378                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 379                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 380                 POSTING_READ(FW_BLC_SELF);
 381         } else if (IS_PINEVIEW(dev_priv)) {
 382                 val = I915_READ(DSPFW3);
 383                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 384                 if (enable)
 385                         val |= PINEVIEW_SELF_REFRESH_EN;
 386                 else
 387                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 388                 I915_WRITE(DSPFW3, val);
 389                 POSTING_READ(DSPFW3);
 390         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 391                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 392                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 393                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 394                 I915_WRITE(FW_BLC_SELF, val);
 395                 POSTING_READ(FW_BLC_SELF);
 396         } else if (IS_I915GM(dev_priv)) {
 397                 /*
 398                  * FIXME can't find a bit like this for 915G, and
 399                  * and yet it does have the related watermark in
 400                  * FW_BLC_SELF. What's going on?
 401                  */
 402                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 403                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 404                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 405                 I915_WRITE(INSTPM, val);
 406                 POSTING_READ(INSTPM);
 407         } else {
 408                 return false;
 409         }
 410
 411         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 412
 413         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 414                     enableddisabled(enable),
 415                     enableddisabled(was_enabled));
 416
 417         return was_enabled;
 418 }
 419
 420 /**
 421  * intel_set_memory_cxsr - Configure CxSR state
 422  * @dev_priv: i915 device
 423  * @enable: Allow vs. disallow CxSR
 424  *
 425  * Allow or disallow the system to enter a special CxSR
 426  * (C-state self refresh) state. What typically happens in CxSR mode
 427  * is that several display FIFOs may get combined into a single larger
 428  * FIFO for a particular plane (so called max FIFO mode) to allow the
 429  * system to defer memory fetches longer, and the memory will enter
 430  * self refresh.
 431  *
 432  * Note that enabling CxSR does not guarantee that the system enter
 433  * this special mode, nor does it guarantee that the system stays
 434  * in that mode once entered. So this just allows/disallows the system
 435  * to autonomously utilize the CxSR mode. Other factors such as core
 436  * C-states will affect when/if the system actually enters/exits the
 437  * CxSR mode.
 438  *
 439  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 440  * and the system is free to enter/exit memory self refresh at any time
 441  * even when the use of CxSR has been disallowed.
 442  *
 443  * While the system is actually in the CxSR/max FIFO mode, some plane
 444  * control registers will not get latched on vblank. Thus in order to
 445  * guarantee the system will respond to changes in the plane registers
 446  * we must always disallow CxSR prior to making changes to those registers.
 447  * Unfortunately the system will re-evaluate the CxSR conditions at
 448  * frame start which happens after vblank start (which is when the plane
 449  * registers would get latched), so we can't proceed with the plane update
 450  * during the same frame where we disallowed CxSR.
 451  *
 452  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 453  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 454  * the hardware w.r.t. HPLL SR when writing to plane registers.
 455  * Disallowing just CxSR is sufficient.
 456  */
 457 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 458 {
 459         bool ret;
 460
 461         mutex_lock(&dev_priv->wm.wm_mutex);
 462         ret = _intel_set_memory_cxsr(dev_priv, enable);
 463         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 464                 dev_priv->wm.vlv.cxsr = enable;
 465         else if (IS_G4X(dev_priv))
 466                 dev_priv->wm.g4x.cxsr = enable;
 467         mutex_unlock(&dev_priv->wm.wm_mutex);
 468
 469         return ret;
 470 }
 471
 472 /*
 473  * Latency for FIFO fetches is dependent on several factors:
 474  *   - memory configuration (speed, channels)
 475  *   - chipset
 476  *   - current MCH state
 477  * It can be fairly high in some situations, so here we assume a fairly
 478  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 479  * set this value too high, the FIFO will fetch frequently to stay full)
 480  * and power consumption (set it too low to save power and we might see
 481  * FIFO underruns and display "flicker").
 482  *
 483  * A value of 5us seems to be a good balance; safe for very low end
 484  * platforms but not overly aggressive on lower latency configs.
 485  */
 486 static const int pessimal_latency_ns = 5000;
 487
 488 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 489         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 490
 491 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 492 {
 493         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 494         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 495         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 496         enum pipe pipe = crtc->pipe;
 497         int sprite0_start, sprite1_start;
 498         u32 dsparb, dsparb2, dsparb3;
 499
 500         switch (pipe) {
 501         case PIPE_A:
 502                 dsparb = I915_READ(DSPARB);
 503                 dsparb2 = I915_READ(DSPARB2);
 504                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 505                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 506                 break;
 507         case PIPE_B:
 508                 dsparb = I915_READ(DSPARB);
 509                 dsparb2 = I915_READ(DSPARB2);
 510                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 511                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 512                 break;
 513         case PIPE_C:
 514                 dsparb2 = I915_READ(DSPARB2);
 515                 dsparb3 = I915_READ(DSPARB3);
 516                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 517                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 518                 break;
 519         default:
 520                 MISSING_CASE(pipe);
 521                 return;
 522         }
 523
 524         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 525         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 526         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 527         fifo_state->plane[PLANE_CURSOR] = 63;
 528 }
 529
 530 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 531                               enum i9xx_plane_id i9xx_plane)
 532 {
 533         u32 dsparb = I915_READ(DSPARB);
 534         int size;
 535
 536         size = dsparb & 0x7f;
 537         if (i9xx_plane == PLANE_B)
 538                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 539
 540         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 541                     dsparb, plane_name(i9xx_plane), size);
 542
 543         return size;
 544 }
 545
 546 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 547                               enum i9xx_plane_id i9xx_plane)
 548 {
 549         u32 dsparb = I915_READ(DSPARB);
 550         int size;
 551
 552         size = dsparb & 0x1ff;
 553         if (i9xx_plane == PLANE_B)
 554                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 555         size >>= 1; /* Convert to cachelines */
 556
 557         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 558                     dsparb, plane_name(i9xx_plane), size);
 559
 560         return size;
 561 }
 562
 563 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 564                               enum i9xx_plane_id i9xx_plane)
 565 {
 566         u32 dsparb = I915_READ(DSPARB);
 567         int size;
 568
 569         size = dsparb & 0x7f;
 570         size >>= 2; /* Convert to cachelines */
 571
 572         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 573                     dsparb, plane_name(i9xx_plane), size);
 574
 575         return size;
 576 }
 577
 578 /* Pineview has different values for various configs */
 579 static const struct intel_watermark_params pnv_display_wm = {
 580         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 581         .max_wm = PINEVIEW_MAX_WM,
 582         .default_wm = PINEVIEW_DFT_WM,
 583         .guard_size = PINEVIEW_GUARD_WM,
 584         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 585 };
 586
 587 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 588         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 589         .max_wm = PINEVIEW_MAX_WM,
 590         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 591         .guard_size = PINEVIEW_GUARD_WM,
 592         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 593 };
 594
 595 static const struct intel_watermark_params pnv_cursor_wm = {
 596         .fifo_size = PINEVIEW_CURSOR_FIFO,
 597         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 598         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 599         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 600         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 601 };
 602
 603 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 604         .fifo_size = PINEVIEW_CURSOR_FIFO,
 605         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 606         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 607         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 608         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 609 };
 610
 611 static const struct intel_watermark_params i965_cursor_wm_info = {
 612         .fifo_size = I965_CURSOR_FIFO,
 613         .max_wm = I965_CURSOR_MAX_WM,
 614         .default_wm = I965_CURSOR_DFT_WM,
 615         .guard_size = 2,
 616         .cacheline_size = I915_FIFO_LINE_SIZE,
 617 };
 618
 619 static const struct intel_watermark_params i945_wm_info = {
 620         .fifo_size = I945_FIFO_SIZE,
 621         .max_wm = I915_MAX_WM,
 622         .default_wm = 1,
 623         .guard_size = 2,
 624         .cacheline_size = I915_FIFO_LINE_SIZE,
 625 };
 626
 627 static const struct intel_watermark_params i915_wm_info = {
 628         .fifo_size = I915_FIFO_SIZE,
 629         .max_wm = I915_MAX_WM,
 630         .default_wm = 1,
 631         .guard_size = 2,
 632         .cacheline_size = I915_FIFO_LINE_SIZE,
 633 };
 634
 635 static const struct intel_watermark_params i830_a_wm_info = {
 636         .fifo_size = I855GM_FIFO_SIZE,
 637         .max_wm = I915_MAX_WM,
 638         .default_wm = 1,
 639         .guard_size = 2,
 640         .cacheline_size = I830_FIFO_LINE_SIZE,
 641 };
 642
 643 static const struct intel_watermark_params i830_bc_wm_info = {
 644         .fifo_size = I855GM_FIFO_SIZE,
 645         .max_wm = I915_MAX_WM/2,
 646         .default_wm = 1,
 647         .guard_size = 2,
 648         .cacheline_size = I830_FIFO_LINE_SIZE,
 649 };
 650
 651 static const struct intel_watermark_params i845_wm_info = {
 652         .fifo_size = I830_FIFO_SIZE,
 653         .max_wm = I915_MAX_WM,
 654         .default_wm = 1,
 655         .guard_size = 2,
 656         .cacheline_size = I830_FIFO_LINE_SIZE,
 657 };
 658
 659 /**
 660  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 661  * @pixel_rate: Pipe pixel rate in kHz
 662  * @cpp: Plane bytes per pixel
 663  * @latency: Memory wakeup latency in 0.1us units
 664  *
 665  * Compute the watermark using the method 1 or "small buffer"
 666  * formula. The caller may additonally add extra cachelines
 667  * to account for TLB misses and clock crossings.
 668  *
 669  * This method is concerned with the short term drain rate
 670  * of the FIFO, ie. it does not account for blanking periods
 671  * which would effectively reduce the average drain rate across
 672  * a longer period. The name "small" refers to the fact the
 673  * FIFO is relatively small compared to the amount of data
 674  * fetched.
 675  *
 676  * The FIFO level vs. time graph might look something like:
 677  *
 678  *   |\   |\
 679  *   | \  | \
 680  * __---__---__ (- plane active, _ blanking)
 681  * -> time
 682  *
 683  * or perhaps like this:
 684  *
 685  *   |\|\  |\|\
 686  * __----__----__ (- plane active, _ blanking)
 687  * -> time
 688  *
 689  * Returns:
 690  * The watermark in bytes
 691  */
 692 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 693                                      unsigned int cpp,
 694                                      unsigned int latency)
 695 {
 696         u64 ret;
 697
 698         ret = mul_u32_u32(pixel_rate, cpp * latency);
 699         ret = DIV_ROUND_UP_ULL(ret, 10000);
 700
 701         return ret;
 702 }
 703
 704 /**
 705  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 706  * @pixel_rate: Pipe pixel rate in kHz
 707  * @htotal: Pipe horizontal total
 708  * @width: Plane width in pixels
 709  * @cpp: Plane bytes per pixel
 710  * @latency: Memory wakeup latency in 0.1us units
 711  *
 712  * Compute the watermark using the method 2 or "large buffer"
 713  * formula. The caller may additonally add extra cachelines
 714  * to account for TLB misses and clock crossings.
 715  *
 716  * This method is concerned with the long term drain rate
 717  * of the FIFO, ie. it does account for blanking periods
 718  * which effectively reduce the average drain rate across
 719  * a longer period. The name "large" refers to the fact the
 720  * FIFO is relatively large compared to the amount of data
 721  * fetched.
 722  *
 723  * The FIFO level vs. time graph might look something like:
 724  *
 725  *    |\___       |\___
 726  *    |    \___   |    \___
 727  *    |        \  |        \
 728  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 729  * -> time
 730  *
 731  * Returns:
 732  * The watermark in bytes
 733  */
 734 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 735                                      unsigned int htotal,
 736                                      unsigned int width,
 737                                      unsigned int cpp,
 738                                      unsigned int latency)
 739 {
 740         unsigned int ret;
 741
 742         /*
 743          * FIXME remove once all users are computing
 744          * watermarks in the correct place.
 745          */
 746         if (WARN_ON_ONCE(htotal == 0))
 747                 htotal = 1;
 748
 749         ret = (latency * pixel_rate) / (htotal * 10000);
 750         ret = (ret + 1) * width * cpp;
 751
 752         return ret;
 753 }
 754
 755 /**
 756  * intel_calculate_wm - calculate watermark level
 757  * @pixel_rate: pixel clock
 758  * @wm: chip FIFO params
 759  * @fifo_size: size of the FIFO buffer
 760  * @cpp: bytes per pixel
 761  * @latency_ns: memory latency for the platform
 762  *
 763  * Calculate the watermark level (the level at which the display plane will
 764  * start fetching from memory again).  Each chip has a different display
 765  * FIFO size and allocation, so the caller needs to figure that out and pass
 766  * in the correct intel_watermark_params structure.
 767  *
 768  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 769  * on the pixel size.  When it reaches the watermark level, it'll start
 770  * fetching FIFO line sized based chunks from memory until the FIFO fills
 771  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 772  * will occur, and a display engine hang could result.
 773  */
 774 static unsigned int intel_calculate_wm(int pixel_rate,
 775                                        const struct intel_watermark_params *wm,
 776                                        int fifo_size, int cpp,
 777                                        unsigned int latency_ns)
 778 {
 779         int entries, wm_size;
 780
 781         /*
 782          * Note: we need to make sure we don't overflow for various clock &
 783          * latency values.
 784          * clocks go from a few thousand to several hundred thousand.
 785          * latency is usually a few thousand
 786          */
 787         entries = intel_wm_method1(pixel_rate, cpp,
 788                                    latency_ns / 100);
 789         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 790                 wm->guard_size;
 791         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 792
 793         wm_size = fifo_size - entries;
 794         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 795
 796         /* Don't promote wm_size to unsigned... */
 797         if (wm_size > wm->max_wm)
 798                 wm_size = wm->max_wm;
 799         if (wm_size <= 0)
 800                 wm_size = wm->default_wm;
 801
 802         /*
 803          * Bspec seems to indicate that the value shouldn't be lower than
 804          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 805          * Lets go for 8 which is the burst size since certain platforms
 806          * already use a hardcoded 8 (which is what the spec says should be
 807          * done).
 808          */
 809         if (wm_size <= 8)
 810                 wm_size = 8;
 811
 812         return wm_size;
 813 }
 814
 815 static bool is_disabling(int old, int new, int threshold)
 816 {
 817         return old >= threshold && new < threshold;
 818 }
 819
 820 static bool is_enabling(int old, int new, int threshold)
 821 {
 822         return old < threshold && new >= threshold;
 823 }
 824
 825 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 826 {
 827         return dev_priv->wm.max_level + 1;
 828 }
 829
 830 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 831                                    const struct intel_plane_state *plane_state)
 832 {
 833         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 834
 835         /* FIXME check the 'enable' instead */
 836         if (!crtc_state->hw.active)
 837                 return false;
 838
 839         /*
 840          * Treat cursor with fb as always visible since cursor updates
 841          * can happen faster than the vrefresh rate, and the current
 842          * watermark code doesn't handle that correctly. Cursor updates
 843          * which set/clear the fb or change the cursor size are going
 844          * to get throttled by intel_legacy_cursor_update() to work
 845          * around this problem with the watermark code.
 846          */
 847         if (plane->id == PLANE_CURSOR)
 848                 return plane_state->hw.fb != NULL;
 849         else
 850                 return plane_state->uapi.visible;
 851 }
 852
 853 static bool intel_crtc_active(struct intel_crtc *crtc)
 854 {
 855         /* Be paranoid as we can arrive here with only partial
 856          * state retrieved from the hardware during setup.
 857          *
 858          * We can ditch the adjusted_mode.crtc_clock check as soon
 859          * as Haswell has gained clock readout/fastboot support.
 860          *
 861          * We can ditch the crtc->primary->state->fb check as soon as we can
 862          * properly reconstruct framebuffers.
 863          *
 864          * FIXME: The intel_crtc->active here should be switched to
 865          * crtc->state->active once we have proper CRTC states wired up
 866          * for atomic.
 867          */
 868         return crtc->active && crtc->base.primary->state->fb &&
 869                 crtc->config->hw.adjusted_mode.crtc_clock;
 870 }
 871
 872 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 873 {
 874         struct intel_crtc *crtc, *enabled = NULL;
 875
 876         for_each_intel_crtc(&dev_priv->drm, crtc) {
 877                 if (intel_crtc_active(crtc)) {
 878                         if (enabled)
 879                                 return NULL;
 880                         enabled = crtc;
 881                 }
 882         }
 883
 884         return enabled;
 885 }
 886
 887 static void pnv_update_wm(struct intel_crtc *unused_crtc)
 888 {
 889         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 890         struct intel_crtc *crtc;
 891         const struct cxsr_latency *latency;
 892         u32 reg;
 893         unsigned int wm;
 894
 895         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 896                                          dev_priv->is_ddr3,
 897                                          dev_priv->fsb_freq,
 898                                          dev_priv->mem_freq);
 899         if (!latency) {
 900                 drm_dbg_kms(&dev_priv->drm,
 901                             "Unknown FSB/MEM found, disable CxSR\n");
 902                 intel_set_memory_cxsr(dev_priv, false);
 903                 return;
 904         }
 905
 906         crtc = single_enabled_crtc(dev_priv);
 907         if (crtc) {
 908                 const struct drm_display_mode *adjusted_mode =
 909                         &crtc->config->hw.adjusted_mode;
 910                 const struct drm_framebuffer *fb =
 911                         crtc->base.primary->state->fb;
 912                 int cpp = fb->format->cpp[0];
 913                 int clock = adjusted_mode->crtc_clock;
 914
 915                 /* Display SR */
 916                 wm = intel_calculate_wm(clock, &pnv_display_wm,
 917                                         pnv_display_wm.fifo_size,
 918                                         cpp, latency->display_sr);
 919                 reg = I915_READ(DSPFW1);
 920                 reg &= ~DSPFW_SR_MASK;
 921                 reg |= FW_WM(wm, SR);
 922                 I915_WRITE(DSPFW1, reg);
 923                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 924
 925                 /* cursor SR */
 926                 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
 927                                         pnv_display_wm.fifo_size,
 928                                         4, latency->cursor_sr);
 929                 reg = I915_READ(DSPFW3);
 930                 reg &= ~DSPFW_CURSOR_SR_MASK;
 931                 reg |= FW_WM(wm, CURSOR_SR);
 932                 I915_WRITE(DSPFW3, reg);
 933
 934                 /* Display HPLL off SR */
 935                 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
 936                                         pnv_display_hplloff_wm.fifo_size,
 937                                         cpp, latency->display_hpll_disable);
 938                 reg = I915_READ(DSPFW3);
 939                 reg &= ~DSPFW_HPLL_SR_MASK;
 940                 reg |= FW_WM(wm, HPLL_SR);
 941                 I915_WRITE(DSPFW3, reg);
 942
 943                 /* cursor HPLL off SR */
 944                 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
 945                                         pnv_display_hplloff_wm.fifo_size,
 946                                         4, latency->cursor_hpll_disable);
 947                 reg = I915_READ(DSPFW3);
 948                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 949                 reg |= FW_WM(wm, HPLL_CURSOR);
 950                 I915_WRITE(DSPFW3, reg);
 951                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 952
 953                 intel_set_memory_cxsr(dev_priv, true);
 954         } else {
 955                 intel_set_memory_cxsr(dev_priv, false);
 956         }
 957 }
 958
 959 /*
 960  * Documentation says:
 961  * "If the line size is small, the TLB fetches can get in the way of the
 962  *  data fetches, causing some lag in the pixel data return which is not
 963  *  accounted for in the above formulas. The following adjustment only
 964  *  needs to be applied if eight whole lines fit in the buffer at once.
 965  *  The WM is adjusted upwards by the difference between the FIFO size
 966  *  and the size of 8 whole lines. This adjustment is always performed
 967  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 968  */
 969 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 970 {
 971         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 972
 973         return max(0, tlb_miss);
 974 }
 975
 976 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 977                                 const struct g4x_wm_values *wm)
 978 {
 979         enum pipe pipe;
 980
 981         for_each_pipe(dev_priv, pipe)
 982                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 983
 984         I915_WRITE(DSPFW1,
 985                    FW_WM(wm->sr.plane, SR) |
 986                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 987                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 988                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 989         I915_WRITE(DSPFW2,
 990                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 991                    FW_WM(wm->sr.fbc, FBC_SR) |
 992                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 993                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 994                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 995                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 996         I915_WRITE(DSPFW3,
 997                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 998                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 999                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1000                    FW_WM(wm->hpll.plane, HPLL_SR));
1001
1002         POSTING_READ(DSPFW1);
1003 }
1004
1005 #define FW_WM_VLV(value, plane) \
1006         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1007
1008 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1009                                 const struct vlv_wm_values *wm)
1010 {
1011         enum pipe pipe;
1012
1013         for_each_pipe(dev_priv, pipe) {
1014                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1015
1016                 I915_WRITE(VLV_DDL(pipe),
1017                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1018                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1019                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1020                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1021         }
1022
1023         /*
1024          * Zero the (unused) WM1 watermarks, and also clear all the
1025          * high order bits so that there are no out of bounds values
1026          * present in the registers during the reprogramming.
1027          */
1028         I915_WRITE(DSPHOWM, 0);
1029         I915_WRITE(DSPHOWM1, 0);
1030         I915_WRITE(DSPFW4, 0);
1031         I915_WRITE(DSPFW5, 0);
1032         I915_WRITE(DSPFW6, 0);
1033
1034         I915_WRITE(DSPFW1,
1035                    FW_WM(wm->sr.plane, SR) |
1036                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1037                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1038                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1039         I915_WRITE(DSPFW2,
1040                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1041                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1042                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1043         I915_WRITE(DSPFW3,
1044                    FW_WM(wm->sr.cursor, CURSOR_SR));
1045
1046         if (IS_CHERRYVIEW(dev_priv)) {
1047                 I915_WRITE(DSPFW7_CHV,
1048                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1049                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1050                 I915_WRITE(DSPFW8_CHV,
1051                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1052                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1053                 I915_WRITE(DSPFW9_CHV,
1054                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1055                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1056                 I915_WRITE(DSPHOWM,
1057                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1058                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1059                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1060                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1061                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1062                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1063                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1064                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1065                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1066                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1067         } else {
1068                 I915_WRITE(DSPFW7,
1069                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1070                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1071                 I915_WRITE(DSPHOWM,
1072                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1073                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1074                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1075                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1076                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1077                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1078                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1079         }
1080
1081         POSTING_READ(DSPFW1);
1082 }
1083
1084 #undef FW_WM_VLV
1085
1086 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1087 {
1088         /* all latencies in usec */
1089         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1090         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1091         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1092
1093         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1094 }
1095
1096 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1097 {
1098         /*
1099          * DSPCNTR[13] supposedly controls whether the
1100          * primary plane can use the FIFO space otherwise
1101          * reserved for the sprite plane. It's not 100% clear
1102          * what the actual FIFO size is, but it looks like we
1103          * can happily set both primary and sprite watermarks
1104          * up to 127 cachelines. So that would seem to mean
1105          * that either DSPCNTR[13] doesn't do anything, or that
1106          * the total FIFO is >= 256 cachelines in size. Either
1107          * way, we don't seem to have to worry about this
1108          * repartitioning as the maximum watermark value the
1109          * register can hold for each plane is lower than the
1110          * minimum FIFO size.
1111          */
1112         switch (plane_id) {
1113         case PLANE_CURSOR:
1114                 return 63;
1115         case PLANE_PRIMARY:
1116                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1117         case PLANE_SPRITE0:
1118                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1119         default:
1120                 MISSING_CASE(plane_id);
1121                 return 0;
1122         }
1123 }
1124
1125 static int g4x_fbc_fifo_size(int level)
1126 {
1127         switch (level) {
1128         case G4X_WM_LEVEL_SR:
1129                 return 7;
1130         case G4X_WM_LEVEL_HPLL:
1131                 return 15;
1132         default:
1133                 MISSING_CASE(level);
1134                 return 0;
1135         }
1136 }
1137
1138 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1139                           const struct intel_plane_state *plane_state,
1140                           int level)
1141 {
1142         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1143         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1144         const struct drm_display_mode *adjusted_mode =
1145                 &crtc_state->hw.adjusted_mode;
1146         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1147         unsigned int clock, htotal, cpp, width, wm;
1148
1149         if (latency == 0)
1150                 return USHRT_MAX;
1151
1152         if (!intel_wm_plane_visible(crtc_state, plane_state))
1153                 return 0;
1154
1155         cpp = plane_state->hw.fb->format->cpp[0];
1156
1157         /*
1158          * Not 100% sure which way ELK should go here as the
1159          * spec only says CL/CTG should assume 32bpp and BW
1160          * doesn't need to. But as these things followed the
1161          * mobile vs. desktop lines on gen3 as well, let's
1162          * assume ELK doesn't need this.
1163          *
1164          * The spec also fails to list such a restriction for
1165          * the HPLL watermark, which seems a little strange.
1166          * Let's use 32bpp for the HPLL watermark as well.
1167          */
1168         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1169             level != G4X_WM_LEVEL_NORMAL)
1170                 cpp = max(cpp, 4u);
1171
1172         clock = adjusted_mode->crtc_clock;
1173         htotal = adjusted_mode->crtc_htotal;
1174
1175         width = drm_rect_width(&plane_state->uapi.dst);
1176
1177         if (plane->id == PLANE_CURSOR) {
1178                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1179         } else if (plane->id == PLANE_PRIMARY &&
1180                    level == G4X_WM_LEVEL_NORMAL) {
1181                 wm = intel_wm_method1(clock, cpp, latency);
1182         } else {
1183                 unsigned int small, large;
1184
1185                 small = intel_wm_method1(clock, cpp, latency);
1186                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1187
1188                 wm = min(small, large);
1189         }
1190
1191         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1192                               width, cpp);
1193
1194         wm = DIV_ROUND_UP(wm, 64) + 2;
1195
1196         return min_t(unsigned int, wm, USHRT_MAX);
1197 }
1198
1199 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1200                                  int level, enum plane_id plane_id, u16 value)
1201 {
1202         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1203         bool dirty = false;
1204
1205         for (; level < intel_wm_num_levels(dev_priv); level++) {
1206                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1207
1208                 dirty |= raw->plane[plane_id] != value;
1209                 raw->plane[plane_id] = value;
1210         }
1211
1212         return dirty;
1213 }
1214
1215 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1216                                int level, u16 value)
1217 {
1218         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1219         bool dirty = false;
1220
1221         /* NORMAL level doesn't have an FBC watermark */
1222         level = max(level, G4X_WM_LEVEL_SR);
1223
1224         for (; level < intel_wm_num_levels(dev_priv); level++) {
1225                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1226
1227                 dirty |= raw->fbc != value;
1228                 raw->fbc = value;
1229         }
1230
1231         return dirty;
1232 }
1233
1234 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1235                               const struct intel_plane_state *plane_state,
1236                               u32 pri_val);
1237
1238 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1239                                      const struct intel_plane_state *plane_state)
1240 {
1241         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1242         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1243         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1244         enum plane_id plane_id = plane->id;
1245         bool dirty = false;
1246         int level;
1247
1248         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1249                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1250                 if (plane_id == PLANE_PRIMARY)
1251                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1252                 goto out;
1253         }
1254
1255         for (level = 0; level < num_levels; level++) {
1256                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1257                 int wm, max_wm;
1258
1259                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1260                 max_wm = g4x_plane_fifo_size(plane_id, level);
1261
1262                 if (wm > max_wm)
1263                         break;
1264
1265                 dirty |= raw->plane[plane_id] != wm;
1266                 raw->plane[plane_id] = wm;
1267
1268                 if (plane_id != PLANE_PRIMARY ||
1269                     level == G4X_WM_LEVEL_NORMAL)
1270                         continue;
1271
1272                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1273                                         raw->plane[plane_id]);
1274                 max_wm = g4x_fbc_fifo_size(level);
1275
1276                 /*
1277                  * FBC wm is not mandatory as we
1278                  * can always just disable its use.
1279                  */
1280                 if (wm > max_wm)
1281                         wm = USHRT_MAX;
1282
1283                 dirty |= raw->fbc != wm;
1284                 raw->fbc = wm;
1285         }
1286
1287         /* mark watermarks as invalid */
1288         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1289
1290         if (plane_id == PLANE_PRIMARY)
1291                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1292
1293  out:
1294         if (dirty) {
1295                 drm_dbg_kms(&dev_priv->drm,
1296                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1297                             plane->base.name,
1298                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1299                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1300                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1301
1302                 if (plane_id == PLANE_PRIMARY)
1303                         drm_dbg_kms(&dev_priv->drm,
1304                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1305                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1306                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1307         }
1308
1309         return dirty;
1310 }
1311
1312 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1313                                       enum plane_id plane_id, int level)
1314 {
1315         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1316
1317         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1318 }
1319
1320 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1321                                      int level)
1322 {
1323         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1324
1325         if (level > dev_priv->wm.max_level)
1326                 return false;
1327
1328         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1329                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1330                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1331 }
1332
1333 /* mark all levels starting from 'level' as invalid */
1334 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1335                                struct g4x_wm_state *wm_state, int level)
1336 {
1337         if (level <= G4X_WM_LEVEL_NORMAL) {
1338                 enum plane_id plane_id;
1339
1340                 for_each_plane_id_on_crtc(crtc, plane_id)
1341                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1342         }
1343
1344         if (level <= G4X_WM_LEVEL_SR) {
1345                 wm_state->cxsr = false;
1346                 wm_state->sr.cursor = USHRT_MAX;
1347                 wm_state->sr.plane = USHRT_MAX;
1348                 wm_state->sr.fbc = USHRT_MAX;
1349         }
1350
1351         if (level <= G4X_WM_LEVEL_HPLL) {
1352                 wm_state->hpll_en = false;
1353                 wm_state->hpll.cursor = USHRT_MAX;
1354                 wm_state->hpll.plane = USHRT_MAX;
1355                 wm_state->hpll.fbc = USHRT_MAX;
1356         }
1357 }
1358
1359 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1360                                int level)
1361 {
1362         if (level < G4X_WM_LEVEL_SR)
1363                 return false;
1364
1365         if (level >= G4X_WM_LEVEL_SR &&
1366             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1367                 return false;
1368
1369         if (level >= G4X_WM_LEVEL_HPLL &&
1370             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1371                 return false;
1372
1373         return true;
1374 }
1375
1376 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1377 {
1378         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1379         struct intel_atomic_state *state =
1380                 to_intel_atomic_state(crtc_state->uapi.state);
1381         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1382         int num_active_planes = hweight8(crtc_state->active_planes &
1383                                          ~BIT(PLANE_CURSOR));
1384         const struct g4x_pipe_wm *raw;
1385         const struct intel_plane_state *old_plane_state;
1386         const struct intel_plane_state *new_plane_state;
1387         struct intel_plane *plane;
1388         enum plane_id plane_id;
1389         int i, level;
1390         unsigned int dirty = 0;
1391
1392         for_each_oldnew_intel_plane_in_state(state, plane,
1393                                              old_plane_state,
1394                                              new_plane_state, i) {
1395                 if (new_plane_state->hw.crtc != &crtc->base &&
1396                     old_plane_state->hw.crtc != &crtc->base)
1397                         continue;
1398
1399                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1400                         dirty |= BIT(plane->id);
1401         }
1402
1403         if (!dirty)
1404                 return 0;
1405
1406         level = G4X_WM_LEVEL_NORMAL;
1407         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1408                 goto out;
1409
1410         raw = &crtc_state->wm.g4x.raw[level];
1411         for_each_plane_id_on_crtc(crtc, plane_id)
1412                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1413
1414         level = G4X_WM_LEVEL_SR;
1415         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1416                 goto out;
1417
1418         raw = &crtc_state->wm.g4x.raw[level];
1419         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1420         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1421         wm_state->sr.fbc = raw->fbc;
1422
1423         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1424
1425         level = G4X_WM_LEVEL_HPLL;
1426         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1427                 goto out;
1428
1429         raw = &crtc_state->wm.g4x.raw[level];
1430         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1431         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1432         wm_state->hpll.fbc = raw->fbc;
1433
1434         wm_state->hpll_en = wm_state->cxsr;
1435
1436         level++;
1437
1438  out:
1439         if (level == G4X_WM_LEVEL_NORMAL)
1440                 return -EINVAL;
1441
1442         /* invalidate the higher levels */
1443         g4x_invalidate_wms(crtc, wm_state, level);
1444
1445         /*
1446          * Determine if the FBC watermark(s) can be used. IF
1447          * this isn't the case we prefer to disable the FBC
1448          * watermark(s) rather than disable the SR/HPLL
1449          * level(s) entirely. 'level-1' is the highest valid
1450          * level here.
1451          */
1452         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1453
1454         return 0;
1455 }
1456
1457 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1458 {
1459         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1460         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1461         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1462         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1463         struct intel_atomic_state *intel_state =
1464                 to_intel_atomic_state(new_crtc_state->uapi.state);
1465         const struct intel_crtc_state *old_crtc_state =
1466                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1467         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1468         enum plane_id plane_id;
1469
1470         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1471                 *intermediate = *optimal;
1472
1473                 intermediate->cxsr = false;
1474                 intermediate->hpll_en = false;
1475                 goto out;
1476         }
1477
1478         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1479                 !new_crtc_state->disable_cxsr;
1480         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1481                 !new_crtc_state->disable_cxsr;
1482         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1483
1484         for_each_plane_id_on_crtc(crtc, plane_id) {
1485                 intermediate->wm.plane[plane_id] =
1486                         max(optimal->wm.plane[plane_id],
1487                             active->wm.plane[plane_id]);
1488
1489                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1490                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1491         }
1492
1493         intermediate->sr.plane = max(optimal->sr.plane,
1494                                      active->sr.plane);
1495         intermediate->sr.cursor = max(optimal->sr.cursor,
1496                                       active->sr.cursor);
1497         intermediate->sr.fbc = max(optimal->sr.fbc,
1498                                    active->sr.fbc);
1499
1500         intermediate->hpll.plane = max(optimal->hpll.plane,
1501                                        active->hpll.plane);
1502         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1503                                         active->hpll.cursor);
1504         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1505                                      active->hpll.fbc);
1506
1507         drm_WARN_ON(&dev_priv->drm,
1508                     (intermediate->sr.plane >
1509                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1510                      intermediate->sr.cursor >
1511                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1512                     intermediate->cxsr);
1513         drm_WARN_ON(&dev_priv->drm,
1514                     (intermediate->sr.plane >
1515                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1516                      intermediate->sr.cursor >
1517                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1518                     intermediate->hpll_en);
1519
1520         drm_WARN_ON(&dev_priv->drm,
1521                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1522                     intermediate->fbc_en && intermediate->cxsr);
1523         drm_WARN_ON(&dev_priv->drm,
1524                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1525                     intermediate->fbc_en && intermediate->hpll_en);
1526
1527 out:
1528         /*
1529          * If our intermediate WM are identical to the final WM, then we can
1530          * omit the post-vblank programming; only update if it's different.
1531          */
1532         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1533                 new_crtc_state->wm.need_postvbl_update = true;
1534
1535         return 0;
1536 }
1537
1538 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1539                          struct g4x_wm_values *wm)
1540 {
1541         struct intel_crtc *crtc;
1542         int num_active_pipes = 0;
1543
1544         wm->cxsr = true;
1545         wm->hpll_en = true;
1546         wm->fbc_en = true;
1547
1548         for_each_intel_crtc(&dev_priv->drm, crtc) {
1549                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1550
1551                 if (!crtc->active)
1552                         continue;
1553
1554                 if (!wm_state->cxsr)
1555                         wm->cxsr = false;
1556                 if (!wm_state->hpll_en)
1557                         wm->hpll_en = false;
1558                 if (!wm_state->fbc_en)
1559                         wm->fbc_en = false;
1560
1561                 num_active_pipes++;
1562         }
1563
1564         if (num_active_pipes != 1) {
1565                 wm->cxsr = false;
1566                 wm->hpll_en = false;
1567                 wm->fbc_en = false;
1568         }
1569
1570         for_each_intel_crtc(&dev_priv->drm, crtc) {
1571                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1572                 enum pipe pipe = crtc->pipe;
1573
1574                 wm->pipe[pipe] = wm_state->wm;
1575                 if (crtc->active && wm->cxsr)
1576                         wm->sr = wm_state->sr;
1577                 if (crtc->active && wm->hpll_en)
1578                         wm->hpll = wm_state->hpll;
1579         }
1580 }
1581
1582 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1583 {
1584         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1585         struct g4x_wm_values new_wm = {};
1586
1587         g4x_merge_wm(dev_priv, &new_wm);
1588
1589         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1590                 return;
1591
1592         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1593                 _intel_set_memory_cxsr(dev_priv, false);
1594
1595         g4x_write_wm_values(dev_priv, &new_wm);
1596
1597         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1598                 _intel_set_memory_cxsr(dev_priv, true);
1599
1600         *old_wm = new_wm;
1601 }
1602
1603 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1604                                    struct intel_crtc *crtc)
1605 {
1606         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1607         const struct intel_crtc_state *crtc_state =
1608                 intel_atomic_get_new_crtc_state(state, crtc);
1609
1610         mutex_lock(&dev_priv->wm.wm_mutex);
1611         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1612         g4x_program_watermarks(dev_priv);
1613         mutex_unlock(&dev_priv->wm.wm_mutex);
1614 }
1615
1616 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1617                                     struct intel_crtc *crtc)
1618 {
1619         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1620         const struct intel_crtc_state *crtc_state =
1621                 intel_atomic_get_new_crtc_state(state, crtc);
1622
1623         if (!crtc_state->wm.need_postvbl_update)
1624                 return;
1625
1626         mutex_lock(&dev_priv->wm.wm_mutex);
1627         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1628         g4x_program_watermarks(dev_priv);
1629         mutex_unlock(&dev_priv->wm.wm_mutex);
1630 }
1631
1632 /* latency must be in 0.1us units. */
1633 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1634                                    unsigned int htotal,
1635                                    unsigned int width,
1636                                    unsigned int cpp,
1637                                    unsigned int latency)
1638 {
1639         unsigned int ret;
1640
1641         ret = intel_wm_method2(pixel_rate, htotal,
1642                                width, cpp, latency);
1643         ret = DIV_ROUND_UP(ret, 64);
1644
1645         return ret;
1646 }
1647
1648 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1649 {
1650         /* all latencies in usec */
1651         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1652
1653         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1654
1655         if (IS_CHERRYVIEW(dev_priv)) {
1656                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1657                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1658
1659                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1660         }
1661 }
1662
1663 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1664                                 const struct intel_plane_state *plane_state,
1665                                 int level)
1666 {
1667         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1668         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1669         const struct drm_display_mode *adjusted_mode =
1670                 &crtc_state->hw.adjusted_mode;
1671         unsigned int clock, htotal, cpp, width, wm;
1672
1673         if (dev_priv->wm.pri_latency[level] == 0)
1674                 return USHRT_MAX;
1675
1676         if (!intel_wm_plane_visible(crtc_state, plane_state))
1677                 return 0;
1678
1679         cpp = plane_state->hw.fb->format->cpp[0];
1680         clock = adjusted_mode->crtc_clock;
1681         htotal = adjusted_mode->crtc_htotal;
1682         width = crtc_state->pipe_src_w;
1683
1684         if (plane->id == PLANE_CURSOR) {
1685                 /*
1686                  * FIXME the formula gives values that are
1687                  * too big for the cursor FIFO, and hence we
1688                  * would never be able to use cursors. For
1689                  * now just hardcode the watermark.
1690                  */
1691                 wm = 63;
1692         } else {
1693                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1694                                     dev_priv->wm.pri_latency[level] * 10);
1695         }
1696
1697         return min_t(unsigned int, wm, USHRT_MAX);
1698 }
1699
1700 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1701 {
1702         return (active_planes & (BIT(PLANE_SPRITE0) |
1703                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1704 }
1705
1706 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1707 {
1708         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1709         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1710         const struct g4x_pipe_wm *raw =
1711                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1712         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1713         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1714         int num_active_planes = hweight8(active_planes);
1715         const int fifo_size = 511;
1716         int fifo_extra, fifo_left = fifo_size;
1717         int sprite0_fifo_extra = 0;
1718         unsigned int total_rate;
1719         enum plane_id plane_id;
1720
1721         /*
1722          * When enabling sprite0 after sprite1 has already been enabled
1723          * we tend to get an underrun unless sprite0 already has some
1724          * FIFO space allcoated. Hence we always allocate at least one
1725          * cacheline for sprite0 whenever sprite1 is enabled.
1726          *
1727          * All other plane enable sequences appear immune to this problem.
1728          */
1729         if (vlv_need_sprite0_fifo_workaround(active_planes))
1730                 sprite0_fifo_extra = 1;
1731
1732         total_rate = raw->plane[PLANE_PRIMARY] +
1733                 raw->plane[PLANE_SPRITE0] +
1734                 raw->plane[PLANE_SPRITE1] +
1735                 sprite0_fifo_extra;
1736
1737         if (total_rate > fifo_size)
1738                 return -EINVAL;
1739
1740         if (total_rate == 0)
1741                 total_rate = 1;
1742
1743         for_each_plane_id_on_crtc(crtc, plane_id) {
1744                 unsigned int rate;
1745
1746                 if ((active_planes & BIT(plane_id)) == 0) {
1747                         fifo_state->plane[plane_id] = 0;
1748                         continue;
1749                 }
1750
1751                 rate = raw->plane[plane_id];
1752                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1753                 fifo_left -= fifo_state->plane[plane_id];
1754         }
1755
1756         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1757         fifo_left -= sprite0_fifo_extra;
1758
1759         fifo_state->plane[PLANE_CURSOR] = 63;
1760
1761         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1762
1763         /* spread the remainder evenly */
1764         for_each_plane_id_on_crtc(crtc, plane_id) {
1765                 int plane_extra;
1766
1767                 if (fifo_left == 0)
1768                         break;
1769
1770                 if ((active_planes & BIT(plane_id)) == 0)
1771                         continue;
1772
1773                 plane_extra = min(fifo_extra, fifo_left);
1774                 fifo_state->plane[plane_id] += plane_extra;
1775                 fifo_left -= plane_extra;
1776         }
1777
1778         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1779
1780         /* give it all to the first plane if none are active */
1781         if (active_planes == 0) {
1782                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1783                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1784         }
1785
1786         return 0;
1787 }
1788
1789 /* mark all levels starting from 'level' as invalid */
1790 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1791                                struct vlv_wm_state *wm_state, int level)
1792 {
1793         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1794
1795         for (; level < intel_wm_num_levels(dev_priv); level++) {
1796                 enum plane_id plane_id;
1797
1798                 for_each_plane_id_on_crtc(crtc, plane_id)
1799                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1800
1801                 wm_state->sr[level].cursor = USHRT_MAX;
1802                 wm_state->sr[level].plane = USHRT_MAX;
1803         }
1804 }
1805
1806 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1807 {
1808         if (wm > fifo_size)
1809                 return USHRT_MAX;
1810         else
1811                 return fifo_size - wm;
1812 }
1813
1814 /*
1815  * Starting from 'level' set all higher
1816  * levels to 'value' in the "raw" watermarks.
1817  */
1818 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1819                                  int level, enum plane_id plane_id, u16 value)
1820 {
1821         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1822         int num_levels = intel_wm_num_levels(dev_priv);
1823         bool dirty = false;
1824
1825         for (; level < num_levels; level++) {
1826                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1827
1828                 dirty |= raw->plane[plane_id] != value;
1829                 raw->plane[plane_id] = value;
1830         }
1831
1832         return dirty;
1833 }
1834
1835 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1836                                      const struct intel_plane_state *plane_state)
1837 {
1838         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1839         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1840         enum plane_id plane_id = plane->id;
1841         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1842         int level;
1843         bool dirty = false;
1844
1845         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1846                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1847                 goto out;
1848         }
1849
1850         for (level = 0; level < num_levels; level++) {
1851                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1852                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1853                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1854
1855                 if (wm > max_wm)
1856                         break;
1857
1858                 dirty |= raw->plane[plane_id] != wm;
1859                 raw->plane[plane_id] = wm;
1860         }
1861
1862         /* mark all higher levels as invalid */
1863         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1864
1865 out:
1866         if (dirty)
1867                 drm_dbg_kms(&dev_priv->drm,
1868                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1869                             plane->base.name,
1870                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1871                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1872                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1873
1874         return dirty;
1875 }
1876
1877 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1878                                       enum plane_id plane_id, int level)
1879 {
1880         const struct g4x_pipe_wm *raw =
1881                 &crtc_state->wm.vlv.raw[level];
1882         const struct vlv_fifo_state *fifo_state =
1883                 &crtc_state->wm.vlv.fifo_state;
1884
1885         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1886 }
1887
1888 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1889 {
1890         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1891                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1892                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1893                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1894 }
1895
1896 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1897 {
1898         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1899         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1900         struct intel_atomic_state *state =
1901                 to_intel_atomic_state(crtc_state->uapi.state);
1902         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1903         const struct vlv_fifo_state *fifo_state =
1904                 &crtc_state->wm.vlv.fifo_state;
1905         int num_active_planes = hweight8(crtc_state->active_planes &
1906                                          ~BIT(PLANE_CURSOR));
1907         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1908         const struct intel_plane_state *old_plane_state;
1909         const struct intel_plane_state *new_plane_state;
1910         struct intel_plane *plane;
1911         enum plane_id plane_id;
1912         int level, ret, i;
1913         unsigned int dirty = 0;
1914
1915         for_each_oldnew_intel_plane_in_state(state, plane,
1916                                              old_plane_state,
1917                                              new_plane_state, i) {
1918                 if (new_plane_state->hw.crtc != &crtc->base &&
1919                     old_plane_state->hw.crtc != &crtc->base)
1920                         continue;
1921
1922                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1923                         dirty |= BIT(plane->id);
1924         }
1925
1926         /*
1927          * DSPARB registers may have been reset due to the
1928          * power well being turned off. Make sure we restore
1929          * them to a consistent state even if no primary/sprite
1930          * planes are initially active.
1931          */
1932         if (needs_modeset)
1933                 crtc_state->fifo_changed = true;
1934
1935         if (!dirty)
1936                 return 0;
1937
1938         /* cursor changes don't warrant a FIFO recompute */
1939         if (dirty & ~BIT(PLANE_CURSOR)) {
1940                 const struct intel_crtc_state *old_crtc_state =
1941                         intel_atomic_get_old_crtc_state(state, crtc);
1942                 const struct vlv_fifo_state *old_fifo_state =
1943                         &old_crtc_state->wm.vlv.fifo_state;
1944
1945                 ret = vlv_compute_fifo(crtc_state);
1946                 if (ret)
1947                         return ret;
1948
1949                 if (needs_modeset ||
1950                     memcmp(old_fifo_state, fifo_state,
1951                            sizeof(*fifo_state)) != 0)
1952                         crtc_state->fifo_changed = true;
1953         }
1954
1955         /* initially allow all levels */
1956         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1957         /*
1958          * Note that enabling cxsr with no primary/sprite planes
1959          * enabled can wedge the pipe. Hence we only allow cxsr
1960          * with exactly one enabled primary/sprite plane.
1961          */
1962         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1963
1964         for (level = 0; level < wm_state->num_levels; level++) {
1965                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1966                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1967
1968                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1969                         break;
1970
1971                 for_each_plane_id_on_crtc(crtc, plane_id) {
1972                         wm_state->wm[level].plane[plane_id] =
1973                                 vlv_invert_wm_value(raw->plane[plane_id],
1974                                                     fifo_state->plane[plane_id]);
1975                 }
1976
1977                 wm_state->sr[level].plane =
1978                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1979                                                  raw->plane[PLANE_SPRITE0],
1980                                                  raw->plane[PLANE_SPRITE1]),
1981                                             sr_fifo_size);
1982
1983                 wm_state->sr[level].cursor =
1984                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1985                                             63);
1986         }
1987
1988         if (level == 0)
1989                 return -EINVAL;
1990
1991         /* limit to only levels we can actually handle */
1992         wm_state->num_levels = level;
1993
1994         /* invalidate the higher levels */
1995         vlv_invalidate_wms(crtc, wm_state, level);
1996
1997         return 0;
1998 }
1999
2000 #define VLV_FIFO(plane, value) \
2001         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2002
2003 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2004                                    struct intel_crtc *crtc)
2005 {
2006         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2007         struct intel_uncore *uncore = &dev_priv->uncore;
2008         const struct intel_crtc_state *crtc_state =
2009                 intel_atomic_get_new_crtc_state(state, crtc);
2010         const struct vlv_fifo_state *fifo_state =
2011                 &crtc_state->wm.vlv.fifo_state;
2012         int sprite0_start, sprite1_start, fifo_size;
2013         u32 dsparb, dsparb2, dsparb3;
2014
2015         if (!crtc_state->fifo_changed)
2016                 return;
2017
2018         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2019         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2020         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2021
2022         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2023         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2024
2025         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2026
2027         /*
2028          * uncore.lock serves a double purpose here. It allows us to
2029          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2030          * it protects the DSPARB registers from getting clobbered by
2031          * parallel updates from multiple pipes.
2032          *
2033          * intel_pipe_update_start() has already disabled interrupts
2034          * for us, so a plain spin_lock() is sufficient here.
2035          */
2036         spin_lock(&uncore->lock);
2037
2038         switch (crtc->pipe) {
2039         case PIPE_A:
2040                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2041                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2042
2043                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2044                             VLV_FIFO(SPRITEB, 0xff));
2045                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2046                            VLV_FIFO(SPRITEB, sprite1_start));
2047
2048                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2049                              VLV_FIFO(SPRITEB_HI, 0x1));
2050                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2051                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2052
2053                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2054                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2055                 break;
2056         case PIPE_B:
2057                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2058                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2059
2060                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2061                             VLV_FIFO(SPRITED, 0xff));
2062                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2063                            VLV_FIFO(SPRITED, sprite1_start));
2064
2065                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2066                              VLV_FIFO(SPRITED_HI, 0xff));
2067                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2068                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2069
2070                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2071                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2072                 break;
2073         case PIPE_C:
2074                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2075                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2076
2077                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2078                              VLV_FIFO(SPRITEF, 0xff));
2079                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2080                             VLV_FIFO(SPRITEF, sprite1_start));
2081
2082                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2083                              VLV_FIFO(SPRITEF_HI, 0xff));
2084                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2085                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2086
2087                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2088                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2089                 break;
2090         default:
2091                 break;
2092         }
2093
2094         intel_uncore_posting_read_fw(uncore, DSPARB);
2095
2096         spin_unlock(&uncore->lock);
2097 }
2098
2099 #undef VLV_FIFO
2100
2101 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2102 {
2103         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2104         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2105         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2106         struct intel_atomic_state *intel_state =
2107                 to_intel_atomic_state(new_crtc_state->uapi.state);
2108         const struct intel_crtc_state *old_crtc_state =
2109                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2110         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2111         int level;
2112
2113         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2114                 *intermediate = *optimal;
2115
2116                 intermediate->cxsr = false;
2117                 goto out;
2118         }
2119
2120         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2121         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2122                 !new_crtc_state->disable_cxsr;
2123
2124         for (level = 0; level < intermediate->num_levels; level++) {
2125                 enum plane_id plane_id;
2126
2127                 for_each_plane_id_on_crtc(crtc, plane_id) {
2128                         intermediate->wm[level].plane[plane_id] =
2129                                 min(optimal->wm[level].plane[plane_id],
2130                                     active->wm[level].plane[plane_id]);
2131                 }
2132
2133                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2134                                                     active->sr[level].plane);
2135                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2136                                                      active->sr[level].cursor);
2137         }
2138
2139         vlv_invalidate_wms(crtc, intermediate, level);
2140
2141 out:
2142         /*
2143          * If our intermediate WM are identical to the final WM, then we can
2144          * omit the post-vblank programming; only update if it's different.
2145          */
2146         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2147                 new_crtc_state->wm.need_postvbl_update = true;
2148
2149         return 0;
2150 }
2151
2152 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2153                          struct vlv_wm_values *wm)
2154 {
2155         struct intel_crtc *crtc;
2156         int num_active_pipes = 0;
2157
2158         wm->level = dev_priv->wm.max_level;
2159         wm->cxsr = true;
2160
2161         for_each_intel_crtc(&dev_priv->drm, crtc) {
2162                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2163
2164                 if (!crtc->active)
2165                         continue;
2166
2167                 if (!wm_state->cxsr)
2168                         wm->cxsr = false;
2169
2170                 num_active_pipes++;
2171                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2172         }
2173
2174         if (num_active_pipes != 1)
2175                 wm->cxsr = false;
2176
2177         if (num_active_pipes > 1)
2178                 wm->level = VLV_WM_LEVEL_PM2;
2179
2180         for_each_intel_crtc(&dev_priv->drm, crtc) {
2181                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2182                 enum pipe pipe = crtc->pipe;
2183
2184                 wm->pipe[pipe] = wm_state->wm[wm->level];
2185                 if (crtc->active && wm->cxsr)
2186                         wm->sr = wm_state->sr[wm->level];
2187
2188                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2189                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2190                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2191                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2192         }
2193 }
2194
2195 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2196 {
2197         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2198         struct vlv_wm_values new_wm = {};
2199
2200         vlv_merge_wm(dev_priv, &new_wm);
2201
2202         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2203                 return;
2204
2205         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2206                 chv_set_memory_dvfs(dev_priv, false);
2207
2208         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2209                 chv_set_memory_pm5(dev_priv, false);
2210
2211         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2212                 _intel_set_memory_cxsr(dev_priv, false);
2213
2214         vlv_write_wm_values(dev_priv, &new_wm);
2215
2216         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2217                 _intel_set_memory_cxsr(dev_priv, true);
2218
2219         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2220                 chv_set_memory_pm5(dev_priv, true);
2221
2222         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2223                 chv_set_memory_dvfs(dev_priv, true);
2224
2225         *old_wm = new_wm;
2226 }
2227
2228 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2229                                    struct intel_crtc *crtc)
2230 {
2231         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2232         const struct intel_crtc_state *crtc_state =
2233                 intel_atomic_get_new_crtc_state(state, crtc);
2234
2235         mutex_lock(&dev_priv->wm.wm_mutex);
2236         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2237         vlv_program_watermarks(dev_priv);
2238         mutex_unlock(&dev_priv->wm.wm_mutex);
2239 }
2240
2241 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2242                                     struct intel_crtc *crtc)
2243 {
2244         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2245         const struct intel_crtc_state *crtc_state =
2246                 intel_atomic_get_new_crtc_state(state, crtc);
2247
2248         if (!crtc_state->wm.need_postvbl_update)
2249                 return;
2250
2251         mutex_lock(&dev_priv->wm.wm_mutex);
2252         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2253         vlv_program_watermarks(dev_priv);
2254         mutex_unlock(&dev_priv->wm.wm_mutex);
2255 }
2256
2257 static void i965_update_wm(struct intel_crtc *unused_crtc)
2258 {
2259         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2260         struct intel_crtc *crtc;
2261         int srwm = 1;
2262         int cursor_sr = 16;
2263         bool cxsr_enabled;
2264
2265         /* Calc sr entries for one plane configs */
2266         crtc = single_enabled_crtc(dev_priv);
2267         if (crtc) {
2268                 /* self-refresh has much higher latency */
2269                 static const int sr_latency_ns = 12000;
2270                 const struct drm_display_mode *adjusted_mode =
2271                         &crtc->config->hw.adjusted_mode;
2272                 const struct drm_framebuffer *fb =
2273                         crtc->base.primary->state->fb;
2274                 int clock = adjusted_mode->crtc_clock;
2275                 int htotal = adjusted_mode->crtc_htotal;
2276                 int hdisplay = crtc->config->pipe_src_w;
2277                 int cpp = fb->format->cpp[0];
2278                 int entries;
2279
2280                 entries = intel_wm_method2(clock, htotal,
2281                                            hdisplay, cpp, sr_latency_ns / 100);
2282                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2283                 srwm = I965_FIFO_SIZE - entries;
2284                 if (srwm < 0)
2285                         srwm = 1;
2286                 srwm &= 0x1ff;
2287                 drm_dbg_kms(&dev_priv->drm,
2288                             "self-refresh entries: %d, wm: %d\n",
2289                             entries, srwm);
2290
2291                 entries = intel_wm_method2(clock, htotal,
2292                                            crtc->base.cursor->state->crtc_w, 4,
2293                                            sr_latency_ns / 100);
2294                 entries = DIV_ROUND_UP(entries,
2295                                        i965_cursor_wm_info.cacheline_size) +
2296                         i965_cursor_wm_info.guard_size;
2297
2298                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2299                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2300                         cursor_sr = i965_cursor_wm_info.max_wm;
2301
2302                 drm_dbg_kms(&dev_priv->drm,
2303                             "self-refresh watermark: display plane %d "
2304                             "cursor %d\n", srwm, cursor_sr);
2305
2306                 cxsr_enabled = true;
2307         } else {
2308                 cxsr_enabled = false;
2309                 /* Turn off self refresh if both pipes are enabled */
2310                 intel_set_memory_cxsr(dev_priv, false);
2311         }
2312
2313         drm_dbg_kms(&dev_priv->drm,
2314                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2315                     srwm);
2316
2317         /* 965 has limitations... */
2318         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2319                    FW_WM(8, CURSORB) |
2320                    FW_WM(8, PLANEB) |
2321                    FW_WM(8, PLANEA));
2322         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2323                    FW_WM(8, PLANEC_OLD));
2324         /* update cursor SR watermark */
2325         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2326
2327         if (cxsr_enabled)
2328                 intel_set_memory_cxsr(dev_priv, true);
2329 }
2330
2331 #undef FW_WM
2332
2333 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2334 {
2335         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2336         const struct intel_watermark_params *wm_info;
2337         u32 fwater_lo;
2338         u32 fwater_hi;
2339         int cwm, srwm = 1;
2340         int fifo_size;
2341         int planea_wm, planeb_wm;
2342         struct intel_crtc *crtc, *enabled = NULL;
2343
2344         if (IS_I945GM(dev_priv))
2345                 wm_info = &i945_wm_info;
2346         else if (!IS_GEN(dev_priv, 2))
2347                 wm_info = &i915_wm_info;
2348         else
2349                 wm_info = &i830_a_wm_info;
2350
2351         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2352         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2353         if (intel_crtc_active(crtc)) {
2354                 const struct drm_display_mode *adjusted_mode =
2355                         &crtc->config->hw.adjusted_mode;
2356                 const struct drm_framebuffer *fb =
2357                         crtc->base.primary->state->fb;
2358                 int cpp;
2359
2360                 if (IS_GEN(dev_priv, 2))
2361                         cpp = 4;
2362                 else
2363                         cpp = fb->format->cpp[0];
2364
2365                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2366                                                wm_info, fifo_size, cpp,
2367                                                pessimal_latency_ns);
2368                 enabled = crtc;
2369         } else {
2370                 planea_wm = fifo_size - wm_info->guard_size;
2371                 if (planea_wm > (long)wm_info->max_wm)
2372                         planea_wm = wm_info->max_wm;
2373         }
2374
2375         if (IS_GEN(dev_priv, 2))
2376                 wm_info = &i830_bc_wm_info;
2377
2378         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2379         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2380         if (intel_crtc_active(crtc)) {
2381                 const struct drm_display_mode *adjusted_mode =
2382                         &crtc->config->hw.adjusted_mode;
2383                 const struct drm_framebuffer *fb =
2384                         crtc->base.primary->state->fb;
2385                 int cpp;
2386
2387                 if (IS_GEN(dev_priv, 2))
2388                         cpp = 4;
2389                 else
2390                         cpp = fb->format->cpp[0];
2391
2392                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2393                                                wm_info, fifo_size, cpp,
2394                                                pessimal_latency_ns);
2395                 if (enabled == NULL)
2396                         enabled = crtc;
2397                 else
2398                         enabled = NULL;
2399         } else {
2400                 planeb_wm = fifo_size - wm_info->guard_size;
2401                 if (planeb_wm > (long)wm_info->max_wm)
2402                         planeb_wm = wm_info->max_wm;
2403         }
2404
2405         drm_dbg_kms(&dev_priv->drm,
2406                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2407
2408         if (IS_I915GM(dev_priv) && enabled) {
2409                 struct drm_i915_gem_object *obj;
2410
2411                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2412
2413                 /* self-refresh seems busted with untiled */
2414                 if (!i915_gem_object_is_tiled(obj))
2415                         enabled = NULL;
2416         }
2417
2418         /*
2419          * Overlay gets an aggressive default since video jitter is bad.
2420          */
2421         cwm = 2;
2422
2423         /* Play safe and disable self-refresh before adjusting watermarks. */
2424         intel_set_memory_cxsr(dev_priv, false);
2425
2426         /* Calc sr entries for one plane configs */
2427         if (HAS_FW_BLC(dev_priv) && enabled) {
2428                 /* self-refresh has much higher latency */
2429                 static const int sr_latency_ns = 6000;
2430                 const struct drm_display_mode *adjusted_mode =
2431                         &enabled->config->hw.adjusted_mode;
2432                 const struct drm_framebuffer *fb =
2433                         enabled->base.primary->state->fb;
2434                 int clock = adjusted_mode->crtc_clock;
2435                 int htotal = adjusted_mode->crtc_htotal;
2436                 int hdisplay = enabled->config->pipe_src_w;
2437                 int cpp;
2438                 int entries;
2439
2440                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2441                         cpp = 4;
2442                 else
2443                         cpp = fb->format->cpp[0];
2444
2445                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2446                                            sr_latency_ns / 100);
2447                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2448                 drm_dbg_kms(&dev_priv->drm,
2449                             "self-refresh entries: %d\n", entries);
2450                 srwm = wm_info->fifo_size - entries;
2451                 if (srwm < 0)
2452                         srwm = 1;
2453
2454                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2455                         I915_WRITE(FW_BLC_SELF,
2456                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2457                 else
2458                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2459         }
2460
2461         drm_dbg_kms(&dev_priv->drm,
2462                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2463                      planea_wm, planeb_wm, cwm, srwm);
2464
2465         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2466         fwater_hi = (cwm & 0x1f);
2467
2468         /* Set request length to 8 cachelines per fetch */
2469         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2470         fwater_hi = fwater_hi | (1 << 8);
2471
2472         I915_WRITE(FW_BLC, fwater_lo);
2473         I915_WRITE(FW_BLC2, fwater_hi);
2474
2475         if (enabled)
2476                 intel_set_memory_cxsr(dev_priv, true);
2477 }
2478
2479 static void i845_update_wm(struct intel_crtc *unused_crtc)
2480 {
2481         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2482         struct intel_crtc *crtc;
2483         const struct drm_display_mode *adjusted_mode;
2484         u32 fwater_lo;
2485         int planea_wm;
2486
2487         crtc = single_enabled_crtc(dev_priv);
2488         if (crtc == NULL)
2489                 return;
2490
2491         adjusted_mode = &crtc->config->hw.adjusted_mode;
2492         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2493                                        &i845_wm_info,
2494                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2495                                        4, pessimal_latency_ns);
2496         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2497         fwater_lo |= (3<<8) | planea_wm;
2498
2499         drm_dbg_kms(&dev_priv->drm,
2500                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2501
2502         I915_WRITE(FW_BLC, fwater_lo);
2503 }
2504
2505 /* latency must be in 0.1us units. */
2506 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2507                                    unsigned int cpp,
2508                                    unsigned int latency)
2509 {
2510         unsigned int ret;
2511
2512         ret = intel_wm_method1(pixel_rate, cpp, latency);
2513         ret = DIV_ROUND_UP(ret, 64) + 2;
2514
2515         return ret;
2516 }
2517
2518 /* latency must be in 0.1us units. */
2519 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2520                                    unsigned int htotal,
2521                                    unsigned int width,
2522                                    unsigned int cpp,
2523                                    unsigned int latency)
2524 {
2525         unsigned int ret;
2526
2527         ret = intel_wm_method2(pixel_rate, htotal,
2528                                width, cpp, latency);
2529         ret = DIV_ROUND_UP(ret, 64) + 2;
2530
2531         return ret;
2532 }
2533
2534 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2535 {
2536         /*
2537          * Neither of these should be possible since this function shouldn't be
2538          * called if the CRTC is off or the plane is invisible.  But let's be
2539          * extra paranoid to avoid a potential divide-by-zero if we screw up
2540          * elsewhere in the driver.
2541          */
2542         if (WARN_ON(!cpp))
2543                 return 0;
2544         if (WARN_ON(!horiz_pixels))
2545                 return 0;
2546
2547         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2548 }
2549
2550 struct ilk_wm_maximums {
2551         u16 pri;
2552         u16 spr;
2553         u16 cur;
2554         u16 fbc;
2555 };
2556
2557 /*
2558  * For both WM_PIPE and WM_LP.
2559  * mem_value must be in 0.1us units.
2560  */
2561 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2562                               const struct intel_plane_state *plane_state,
2563                               u32 mem_value, bool is_lp)
2564 {
2565         u32 method1, method2;
2566         int cpp;
2567
2568         if (mem_value == 0)
2569                 return U32_MAX;
2570
2571         if (!intel_wm_plane_visible(crtc_state, plane_state))
2572                 return 0;
2573
2574         cpp = plane_state->hw.fb->format->cpp[0];
2575
2576         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2577
2578         if (!is_lp)
2579                 return method1;
2580
2581         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2582                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2583                                  drm_rect_width(&plane_state->uapi.dst),
2584                                  cpp, mem_value);
2585
2586         return min(method1, method2);
2587 }
2588
2589 /*
2590  * For both WM_PIPE and WM_LP.
2591  * mem_value must be in 0.1us units.
2592  */
2593 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2594                               const struct intel_plane_state *plane_state,
2595                               u32 mem_value)
2596 {
2597         u32 method1, method2;
2598         int cpp;
2599
2600         if (mem_value == 0)
2601                 return U32_MAX;
2602
2603         if (!intel_wm_plane_visible(crtc_state, plane_state))
2604                 return 0;
2605
2606         cpp = plane_state->hw.fb->format->cpp[0];
2607
2608         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2609         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2610                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2611                                  drm_rect_width(&plane_state->uapi.dst),
2612                                  cpp, mem_value);
2613         return min(method1, method2);
2614 }
2615
2616 /*
2617  * For both WM_PIPE and WM_LP.
2618  * mem_value must be in 0.1us units.
2619  */
2620 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2621                               const struct intel_plane_state *plane_state,
2622                               u32 mem_value)
2623 {
2624         int cpp;
2625
2626         if (mem_value == 0)
2627                 return U32_MAX;
2628
2629         if (!intel_wm_plane_visible(crtc_state, plane_state))
2630                 return 0;
2631
2632         cpp = plane_state->hw.fb->format->cpp[0];
2633
2634         return ilk_wm_method2(crtc_state->pixel_rate,
2635                               crtc_state->hw.adjusted_mode.crtc_htotal,
2636                               drm_rect_width(&plane_state->uapi.dst),
2637                               cpp, mem_value);
2638 }
2639
2640 /* Only for WM_LP. */
2641 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2642                               const struct intel_plane_state *plane_state,
2643                               u32 pri_val)
2644 {
2645         int cpp;
2646
2647         if (!intel_wm_plane_visible(crtc_state, plane_state))
2648                 return 0;
2649
2650         cpp = plane_state->hw.fb->format->cpp[0];
2651
2652         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2653                           cpp);
2654 }
2655
2656 static unsigned int
2657 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2658 {
2659         if (INTEL_GEN(dev_priv) >= 8)
2660                 return 3072;
2661         else if (INTEL_GEN(dev_priv) >= 7)
2662                 return 768;
2663         else
2664                 return 512;
2665 }
2666
2667 static unsigned int
2668 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2669                      int level, bool is_sprite)
2670 {
2671         if (INTEL_GEN(dev_priv) >= 8)
2672                 /* BDW primary/sprite plane watermarks */
2673                 return level == 0 ? 255 : 2047;
2674         else if (INTEL_GEN(dev_priv) >= 7)
2675                 /* IVB/HSW primary/sprite plane watermarks */
2676                 return level == 0 ? 127 : 1023;
2677         else if (!is_sprite)
2678                 /* ILK/SNB primary plane watermarks */
2679                 return level == 0 ? 127 : 511;
2680         else
2681                 /* ILK/SNB sprite plane watermarks */
2682                 return level == 0 ? 63 : 255;
2683 }
2684
2685 static unsigned int
2686 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2687 {
2688         if (INTEL_GEN(dev_priv) >= 7)
2689                 return level == 0 ? 63 : 255;
2690         else
2691                 return level == 0 ? 31 : 63;
2692 }
2693
2694 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2695 {
2696         if (INTEL_GEN(dev_priv) >= 8)
2697                 return 31;
2698         else
2699                 return 15;
2700 }
2701
2702 /* Calculate the maximum primary/sprite plane watermark */
2703 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2704                                      int level,
2705                                      const struct intel_wm_config *config,
2706                                      enum intel_ddb_partitioning ddb_partitioning,
2707                                      bool is_sprite)
2708 {
2709         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2710
2711         /* if sprites aren't enabled, sprites get nothing */
2712         if (is_sprite && !config->sprites_enabled)
2713                 return 0;
2714
2715         /* HSW allows LP1+ watermarks even with multiple pipes */
2716         if (level == 0 || config->num_pipes_active > 1) {
2717                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2718
2719                 /*
2720                  * For some reason the non self refresh
2721                  * FIFO size is only half of the self
2722                  * refresh FIFO size on ILK/SNB.
2723                  */
2724                 if (INTEL_GEN(dev_priv) <= 6)
2725                         fifo_size /= 2;
2726         }
2727
2728         if (config->sprites_enabled) {
2729                 /* level 0 is always calculated with 1:1 split */
2730                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2731                         if (is_sprite)
2732                                 fifo_size *= 5;
2733                         fifo_size /= 6;
2734                 } else {
2735                         fifo_size /= 2;
2736                 }
2737         }
2738
2739         /* clamp to max that the registers can hold */
2740         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2741 }
2742
2743 /* Calculate the maximum cursor plane watermark */
2744 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2745                                       int level,
2746                                       const struct intel_wm_config *config)
2747 {
2748         /* HSW LP1+ watermarks w/ multiple pipes */
2749         if (level > 0 && config->num_pipes_active > 1)
2750                 return 64;
2751
2752         /* otherwise just report max that registers can hold */
2753         return ilk_cursor_wm_reg_max(dev_priv, level);
2754 }
2755
2756 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2757                                     int level,
2758                                     const struct intel_wm_config *config,
2759                                     enum intel_ddb_partitioning ddb_partitioning,
2760                                     struct ilk_wm_maximums *max)
2761 {
2762         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2763         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2764         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2765         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2766 }
2767
2768 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2769                                         int level,
2770                                         struct ilk_wm_maximums *max)
2771 {
2772         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2773         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2774         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2775         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2776 }
2777
2778 static bool ilk_validate_wm_level(int level,
2779                                   const struct ilk_wm_maximums *max,
2780                                   struct intel_wm_level *result)
2781 {
2782         bool ret;
2783
2784         /* already determined to be invalid? */
2785         if (!result->enable)
2786                 return false;
2787
2788         result->enable = result->pri_val <= max->pri &&
2789                          result->spr_val <= max->spr &&
2790                          result->cur_val <= max->cur;
2791
2792         ret = result->enable;
2793
2794         /*
2795          * HACK until we can pre-compute everything,
2796          * and thus fail gracefully if LP0 watermarks
2797          * are exceeded...
2798          */
2799         if (level == 0 && !result->enable) {
2800                 if (result->pri_val > max->pri)
2801                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2802                                       level, result->pri_val, max->pri);
2803                 if (result->spr_val > max->spr)
2804                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2805                                       level, result->spr_val, max->spr);
2806                 if (result->cur_val > max->cur)
2807                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2808                                       level, result->cur_val, max->cur);
2809
2810                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2811                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2812                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2813                 result->enable = true;
2814         }
2815
2816         return ret;
2817 }
2818
2819 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2820                                  const struct intel_crtc *crtc,
2821                                  int level,
2822                                  struct intel_crtc_state *crtc_state,
2823                                  const struct intel_plane_state *pristate,
2824                                  const struct intel_plane_state *sprstate,
2825                                  const struct intel_plane_state *curstate,
2826                                  struct intel_wm_level *result)
2827 {
2828         u16 pri_latency = dev_priv->wm.pri_latency[level];
2829         u16 spr_latency = dev_priv->wm.spr_latency[level];
2830         u16 cur_latency = dev_priv->wm.cur_latency[level];
2831
2832         /* WM1+ latency values stored in 0.5us units */
2833         if (level > 0) {
2834                 pri_latency *= 5;
2835                 spr_latency *= 5;
2836                 cur_latency *= 5;
2837         }
2838
2839         if (pristate) {
2840                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2841                                                      pri_latency, level);
2842                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2843         }
2844
2845         if (sprstate)
2846                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2847
2848         if (curstate)
2849                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2850
2851         result->enable = true;
2852 }
2853
2854 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2855                                   u16 wm[8])
2856 {
2857         struct intel_uncore *uncore = &dev_priv->uncore;
2858
2859         if (INTEL_GEN(dev_priv) >= 9) {
2860                 u32 val;
2861                 int ret, i;
2862                 int level, max_level = ilk_wm_max_level(dev_priv);
2863
2864                 /* read the first set of memory latencies[0:3] */
2865                 val = 0; /* data0 to be programmed to 0 for first set */
2866                 ret = sandybridge_pcode_read(dev_priv,
2867                                              GEN9_PCODE_READ_MEM_LATENCY,
2868                                              &val, NULL);
2869
2870                 if (ret) {
2871                         drm_err(&dev_priv->drm,
2872                                 "SKL Mailbox read error = %d\n", ret);
2873                         return;
2874                 }
2875
2876                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2877                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2878                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2879                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2880                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2881                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2882                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2883
2884                 /* read the second set of memory latencies[4:7] */
2885                 val = 1; /* data0 to be programmed to 1 for second set */
2886                 ret = sandybridge_pcode_read(dev_priv,
2887                                              GEN9_PCODE_READ_MEM_LATENCY,
2888                                              &val, NULL);
2889                 if (ret) {
2890                         drm_err(&dev_priv->drm,
2891                                 "SKL Mailbox read error = %d\n", ret);
2892                         return;
2893                 }
2894
2895                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2896                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2897                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2898                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2899                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2900                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2901                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2902
2903                 /*
2904                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2905                  * need to be disabled. We make sure to sanitize the values out
2906                  * of the punit to satisfy this requirement.
2907                  */
2908                 for (level = 1; level <= max_level; level++) {
2909                         if (wm[level] == 0) {
2910                                 for (i = level + 1; i <= max_level; i++)
2911                                         wm[i] = 0;
2912                                 break;
2913                         }
2914                 }
2915
2916                 /*
2917                  * WaWmMemoryReadLatency:skl+,glk
2918                  *
2919                  * punit doesn't take into account the read latency so we need
2920                  * to add 2us to the various latency levels we retrieve from the
2921                  * punit when level 0 response data us 0us.
2922                  */
2923                 if (wm[0] == 0) {
2924                         wm[0] += 2;
2925                         for (level = 1; level <= max_level; level++) {
2926                                 if (wm[level] == 0)
2927                                         break;
2928                                 wm[level] += 2;
2929                         }
2930                 }
2931
2932                 /*
2933                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2934                  * If we could not get dimm info enable this WA to prevent from
2935                  * any underrun. If not able to get Dimm info assume 16GB dimm
2936                  * to avoid any underrun.
2937                  */
2938                 if (dev_priv->dram_info.is_16gb_dimm)
2939                         wm[0] += 1;
2940
2941         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2942                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2943
2944                 wm[0] = (sskpd >> 56) & 0xFF;
2945                 if (wm[0] == 0)
2946                         wm[0] = sskpd & 0xF;
2947                 wm[1] = (sskpd >> 4) & 0xFF;
2948                 wm[2] = (sskpd >> 12) & 0xFF;
2949                 wm[3] = (sskpd >> 20) & 0x1FF;
2950                 wm[4] = (sskpd >> 32) & 0x1FF;
2951         } else if (INTEL_GEN(dev_priv) >= 6) {
2952                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2953
2954                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2955                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2956                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2957                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2958         } else if (INTEL_GEN(dev_priv) >= 5) {
2959                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2960
2961                 /* ILK primary LP0 latency is 700 ns */
2962                 wm[0] = 7;
2963                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2964                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2965         } else {
2966                 MISSING_CASE(INTEL_DEVID(dev_priv));
2967         }
2968 }
2969
2970 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2971                                        u16 wm[5])
2972 {
2973         /* ILK sprite LP0 latency is 1300 ns */
2974         if (IS_GEN(dev_priv, 5))
2975                 wm[0] = 13;
2976 }
2977
2978 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2979                                        u16 wm[5])
2980 {
2981         /* ILK cursor LP0 latency is 1300 ns */
2982         if (IS_GEN(dev_priv, 5))
2983                 wm[0] = 13;
2984 }
2985
2986 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2987 {
2988         /* how many WM levels are we expecting */
2989         if (INTEL_GEN(dev_priv) >= 9)
2990                 return 7;
2991         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2992                 return 4;
2993         else if (INTEL_GEN(dev_priv) >= 6)
2994                 return 3;
2995         else
2996                 return 2;
2997 }
2998
2999 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3000                                    const char *name,
3001                                    const u16 wm[8])
3002 {
3003         int level, max_level = ilk_wm_max_level(dev_priv);
3004
3005         for (level = 0; level <= max_level; level++) {
3006                 unsigned int latency = wm[level];
3007
3008                 if (latency == 0) {
3009                         drm_dbg_kms(&dev_priv->drm,
3010                                     "%s WM%d latency not provided\n",
3011                                     name, level);
3012                         continue;
3013                 }
3014
3015                 /*
3016                  * - latencies are in us on gen9.
3017                  * - before then, WM1+ latency values are in 0.5us units
3018                  */
3019                 if (INTEL_GEN(dev_priv) >= 9)
3020                         latency *= 10;
3021                 else if (level > 0)
3022                         latency *= 5;
3023
3024                 drm_dbg_kms(&dev_priv->drm,
3025                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
3026                             wm[level], latency / 10, latency % 10);
3027         }
3028 }
3029
3030 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3031                                     u16 wm[5], u16 min)
3032 {
3033         int level, max_level = ilk_wm_max_level(dev_priv);
3034
3035         if (wm[0] >= min)
3036                 return false;
3037
3038         wm[0] = max(wm[0], min);
3039         for (level = 1; level <= max_level; level++)
3040                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3041
3042         return true;
3043 }
3044
3045 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3046 {
3047         bool changed;
3048
3049         /*
3050          * The BIOS provided WM memory latency values are often
3051          * inadequate for high resolution displays. Adjust them.
3052          */
3053         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3054                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3055                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3056
3057         if (!changed)
3058                 return;
3059
3060         drm_dbg_kms(&dev_priv->drm,
3061                     "WM latency values increased to avoid potential underruns\n");
3062         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3063         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3064         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3065 }
3066
3067 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3068 {
3069         /*
3070          * On some SNB machines (Thinkpad X220 Tablet at least)
3071          * LP3 usage can cause vblank interrupts to be lost.
3072          * The DEIIR bit will go high but it looks like the CPU
3073          * never gets interrupted.
3074          *
3075          * It's not clear whether other interrupt source could
3076          * be affected or if this is somehow limited to vblank
3077          * interrupts only. To play it safe we disable LP3
3078          * watermarks entirely.
3079          */
3080         if (dev_priv->wm.pri_latency[3] == 0 &&
3081             dev_priv->wm.spr_latency[3] == 0 &&
3082             dev_priv->wm.cur_latency[3] == 0)
3083                 return;
3084
3085         dev_priv->wm.pri_latency[3] = 0;
3086         dev_priv->wm.spr_latency[3] = 0;
3087         dev_priv->wm.cur_latency[3] = 0;
3088
3089         drm_dbg_kms(&dev_priv->drm,
3090                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3091         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3092         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3093         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3094 }
3095
3096 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3097 {
3098         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3099
3100         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3101                sizeof(dev_priv->wm.pri_latency));
3102         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3103                sizeof(dev_priv->wm.pri_latency));
3104
3105         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3106         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3107
3108         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3109         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3110         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3111
3112         if (IS_GEN(dev_priv, 6)) {
3113                 snb_wm_latency_quirk(dev_priv);
3114                 snb_wm_lp3_irq_quirk(dev_priv);
3115         }
3116 }
3117
3118 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3119 {
3120         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3121         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3122 }
3123
3124 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3125                                  struct intel_pipe_wm *pipe_wm)
3126 {
3127         /* LP0 watermark maximums depend on this pipe alone */
3128         const struct intel_wm_config config = {
3129                 .num_pipes_active = 1,
3130                 .sprites_enabled = pipe_wm->sprites_enabled,
3131                 .sprites_scaled = pipe_wm->sprites_scaled,
3132         };
3133         struct ilk_wm_maximums max;
3134
3135         /* LP0 watermarks always use 1/2 DDB partitioning */
3136         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3137
3138         /* At least LP0 must be valid */
3139         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3140                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3141                 return false;
3142         }
3143
3144         return true;
3145 }
3146
3147 /* Compute new watermarks for the pipe */
3148 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3149 {
3150         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3151         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3152         struct intel_pipe_wm *pipe_wm;
3153         struct intel_plane *plane;
3154         const struct intel_plane_state *plane_state;
3155         const struct intel_plane_state *pristate = NULL;
3156         const struct intel_plane_state *sprstate = NULL;
3157         const struct intel_plane_state *curstate = NULL;
3158         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3159         struct ilk_wm_maximums max;
3160
3161         pipe_wm = &crtc_state->wm.ilk.optimal;
3162
3163         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3164                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3165                         pristate = plane_state;
3166                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3167                         sprstate = plane_state;
3168                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3169                         curstate = plane_state;
3170         }
3171
3172         pipe_wm->pipe_enabled = crtc_state->hw.active;
3173         if (sprstate) {
3174                 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3175                 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3176                         (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3177                          drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3178         }
3179
3180         usable_level = max_level;
3181
3182         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3183         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3184                 usable_level = 1;
3185
3186         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3187         if (pipe_wm->sprites_scaled)
3188                 usable_level = 0;
3189
3190         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3191         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3192                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3193
3194         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3195                 return -EINVAL;
3196
3197         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3198
3199         for (level = 1; level <= usable_level; level++) {
3200                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3201
3202                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3203                                      pristate, sprstate, curstate, wm);
3204
3205                 /*
3206                  * Disable any watermark level that exceeds the
3207                  * register maximums since such watermarks are
3208                  * always invalid.
3209                  */
3210                 if (!ilk_validate_wm_level(level, &max, wm)) {
3211                         memset(wm, 0, sizeof(*wm));
3212                         break;
3213                 }
3214         }
3215
3216         return 0;
3217 }
3218
3219 /*
3220  * Build a set of 'intermediate' watermark values that satisfy both the old
3221  * state and the new state.  These can be programmed to the hardware
3222  * immediately.
3223  */
3224 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3225 {
3226         struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3227         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3228         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3229         struct intel_atomic_state *intel_state =
3230                 to_intel_atomic_state(newstate->uapi.state);
3231         const struct intel_crtc_state *oldstate =
3232                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3233         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3234         int level, max_level = ilk_wm_max_level(dev_priv);
3235
3236         /*
3237          * Start with the final, target watermarks, then combine with the
3238          * currently active watermarks to get values that are safe both before
3239          * and after the vblank.
3240          */
3241         *a = newstate->wm.ilk.optimal;
3242         if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3243             intel_state->skip_intermediate_wm)
3244                 return 0;
3245
3246         a->pipe_enabled |= b->pipe_enabled;
3247         a->sprites_enabled |= b->sprites_enabled;
3248         a->sprites_scaled |= b->sprites_scaled;
3249
3250         for (level = 0; level <= max_level; level++) {
3251                 struct intel_wm_level *a_wm = &a->wm[level];
3252                 const struct intel_wm_level *b_wm = &b->wm[level];
3253
3254                 a_wm->enable &= b_wm->enable;
3255                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3256                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3257                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3258                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3259         }
3260
3261         /*
3262          * We need to make sure that these merged watermark values are
3263          * actually a valid configuration themselves.  If they're not,
3264          * there's no safe way to transition from the old state to
3265          * the new state, so we need to fail the atomic transaction.
3266          */
3267         if (!ilk_validate_pipe_wm(dev_priv, a))
3268                 return -EINVAL;
3269
3270         /*
3271          * If our intermediate WM are identical to the final WM, then we can
3272          * omit the post-vblank programming; only update if it's different.
3273          */
3274         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3275                 newstate->wm.need_postvbl_update = true;
3276
3277         return 0;
3278 }
3279
3280 /*
3281  * Merge the watermarks from all active pipes for a specific level.
3282  */
3283 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3284                                int level,
3285                                struct intel_wm_level *ret_wm)
3286 {
3287         const struct intel_crtc *intel_crtc;
3288
3289         ret_wm->enable = true;
3290
3291         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3292                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3293                 const struct intel_wm_level *wm = &active->wm[level];
3294
3295                 if (!active->pipe_enabled)
3296                         continue;
3297
3298                 /*
3299                  * The watermark values may have been used in the past,
3300                  * so we must maintain them in the registers for some
3301                  * time even if the level is now disabled.
3302                  */
3303                 if (!wm->enable)
3304                         ret_wm->enable = false;
3305
3306                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3307                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3308                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3309                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3310         }
3311 }
3312
3313 /*
3314  * Merge all low power watermarks for all active pipes.
3315  */
3316 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3317                          const struct intel_wm_config *config,
3318                          const struct ilk_wm_maximums *max,
3319                          struct intel_pipe_wm *merged)
3320 {
3321         int level, max_level = ilk_wm_max_level(dev_priv);
3322         int last_enabled_level = max_level;
3323
3324         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3325         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3326             config->num_pipes_active > 1)
3327                 last_enabled_level = 0;
3328
3329         /* ILK: FBC WM must be disabled always */
3330         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3331
3332         /* merge each WM1+ level */
3333         for (level = 1; level <= max_level; level++) {
3334                 struct intel_wm_level *wm = &merged->wm[level];
3335
3336                 ilk_merge_wm_level(dev_priv, level, wm);
3337
3338                 if (level > last_enabled_level)
3339                         wm->enable = false;
3340                 else if (!ilk_validate_wm_level(level, max, wm))
3341                         /* make sure all following levels get disabled */
3342                         last_enabled_level = level - 1;
3343
3344                 /*
3345                  * The spec says it is preferred to disable
3346                  * FBC WMs instead of disabling a WM level.
3347                  */
3348                 if (wm->fbc_val > max->fbc) {
3349                         if (wm->enable)
3350                                 merged->fbc_wm_enabled = false;
3351                         wm->fbc_val = 0;
3352                 }
3353         }
3354
3355         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3356         /*
3357          * FIXME this is racy. FBC might get enabled later.
3358          * What we should check here is whether FBC can be
3359          * enabled sometime later.
3360          */
3361         if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3362             intel_fbc_is_active(dev_priv)) {
3363                 for (level = 2; level <= max_level; level++) {
3364                         struct intel_wm_level *wm = &merged->wm[level];
3365
3366                         wm->enable = false;
3367                 }
3368         }
3369 }
3370
3371 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3372 {
3373         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3374         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3375 }
3376
3377 /* The value we need to program into the WM_LPx latency field */
3378 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3379                                       int level)
3380 {
3381         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3382                 return 2 * level;
3383         else
3384                 return dev_priv->wm.pri_latency[level];
3385 }
3386
3387 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3388                                    const struct intel_pipe_wm *merged,
3389                                    enum intel_ddb_partitioning partitioning,
3390                                    struct ilk_wm_values *results)
3391 {
3392         struct intel_crtc *intel_crtc;
3393         int level, wm_lp;
3394
3395         results->enable_fbc_wm = merged->fbc_wm_enabled;
3396         results->partitioning = partitioning;
3397
3398         /* LP1+ register values */
3399         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3400                 const struct intel_wm_level *r;
3401
3402                 level = ilk_wm_lp_to_level(wm_lp, merged);
3403
3404                 r = &merged->wm[level];
3405
3406                 /*
3407                  * Maintain the watermark values even if the level is
3408                  * disabled. Doing otherwise could cause underruns.
3409                  */
3410                 results->wm_lp[wm_lp - 1] =
3411                         (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3412                         (r->pri_val << WM1_LP_SR_SHIFT) |
3413                         r->cur_val;
3414
3415                 if (r->enable)
3416                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3417
3418                 if (INTEL_GEN(dev_priv) >= 8)
3419                         results->wm_lp[wm_lp - 1] |=
3420                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3421                 else
3422                         results->wm_lp[wm_lp - 1] |=
3423                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3424
3425                 /*
3426                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3427                  * level is disabled. Doing otherwise could cause underruns.
3428                  */
3429                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3430                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3431                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3432                 } else
3433                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3434         }
3435
3436         /* LP0 register values */
3437         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3438                 enum pipe pipe = intel_crtc->pipe;
3439                 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3440                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3441
3442                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3443                         continue;
3444
3445                 results->wm_pipe[pipe] =
3446                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3447                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3448                         r->cur_val;
3449         }
3450 }
3451
3452 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3453  * case both are at the same level. Prefer r1 in case they're the same. */
3454 static struct intel_pipe_wm *
3455 ilk_find_best_result(struct drm_i915_private *dev_priv,
3456                      struct intel_pipe_wm *r1,
3457                      struct intel_pipe_wm *r2)
3458 {
3459         int level, max_level = ilk_wm_max_level(dev_priv);
3460         int level1 = 0, level2 = 0;
3461
3462         for (level = 1; level <= max_level; level++) {
3463                 if (r1->wm[level].enable)
3464                         level1 = level;
3465                 if (r2->wm[level].enable)
3466                         level2 = level;
3467         }
3468
3469         if (level1 == level2) {
3470                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3471                         return r2;
3472                 else
3473                         return r1;
3474         } else if (level1 > level2) {
3475                 return r1;
3476         } else {
3477                 return r2;
3478         }
3479 }
3480
3481 /* dirty bits used to track which watermarks need changes */
3482 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3483 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3484 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3485 #define WM_DIRTY_FBC (1 << 24)
3486 #define WM_DIRTY_DDB (1 << 25)
3487
3488 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3489                                          const struct ilk_wm_values *old,
3490                                          const struct ilk_wm_values *new)
3491 {
3492         unsigned int dirty = 0;
3493         enum pipe pipe;
3494         int wm_lp;
3495
3496         for_each_pipe(dev_priv, pipe) {
3497                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3498                         dirty |= WM_DIRTY_PIPE(pipe);
3499                         /* Must disable LP1+ watermarks too */
3500                         dirty |= WM_DIRTY_LP_ALL;
3501                 }
3502         }
3503
3504         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3505                 dirty |= WM_DIRTY_FBC;
3506                 /* Must disable LP1+ watermarks too */
3507                 dirty |= WM_DIRTY_LP_ALL;
3508         }
3509
3510         if (old->partitioning != new->partitioning) {
3511                 dirty |= WM_DIRTY_DDB;
3512                 /* Must disable LP1+ watermarks too */
3513                 dirty |= WM_DIRTY_LP_ALL;
3514         }
3515
3516         /* LP1+ watermarks already deemed dirty, no need to continue */
3517         if (dirty & WM_DIRTY_LP_ALL)
3518                 return dirty;
3519
3520         /* Find the lowest numbered LP1+ watermark in need of an update... */
3521         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3522                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3523                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3524                         break;
3525         }
3526
3527         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3528         for (; wm_lp <= 3; wm_lp++)
3529                 dirty |= WM_DIRTY_LP(wm_lp);
3530
3531         return dirty;
3532 }
3533
3534 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3535                                unsigned int dirty)
3536 {
3537         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3538         bool changed = false;
3539
3540         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3541                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3542                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3543                 changed = true;
3544         }
3545         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3546                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3547                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3548                 changed = true;
3549         }
3550         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3551                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3552                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3553                 changed = true;
3554         }
3555
3556         /*
3557          * Don't touch WM1S_LP_EN here.
3558          * Doing so could cause underruns.
3559          */
3560
3561         return changed;
3562 }
3563
3564 /*
3565  * The spec says we shouldn't write when we don't need, because every write
3566  * causes WMs to be re-evaluated, expending some power.
3567  */
3568 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3569                                 struct ilk_wm_values *results)
3570 {
3571         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3572         unsigned int dirty;
3573         u32 val;
3574
3575         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3576         if (!dirty)
3577                 return;
3578
3579         _ilk_disable_lp_wm(dev_priv, dirty);
3580
3581         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3582                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3583         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3584                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3585         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3586                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3587
3588         if (dirty & WM_DIRTY_DDB) {
3589                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3590                         val = I915_READ(WM_MISC);
3591                         if (results->partitioning == INTEL_DDB_PART_1_2)
3592                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3593                         else
3594                                 val |= WM_MISC_DATA_PARTITION_5_6;
3595                         I915_WRITE(WM_MISC, val);
3596                 } else {
3597                         val = I915_READ(DISP_ARB_CTL2);
3598                         if (results->partitioning == INTEL_DDB_PART_1_2)
3599                                 val &= ~DISP_DATA_PARTITION_5_6;
3600                         else
3601                                 val |= DISP_DATA_PARTITION_5_6;
3602                         I915_WRITE(DISP_ARB_CTL2, val);
3603                 }
3604         }
3605
3606         if (dirty & WM_DIRTY_FBC) {
3607                 val = I915_READ(DISP_ARB_CTL);
3608                 if (results->enable_fbc_wm)
3609                         val &= ~DISP_FBC_WM_DIS;
3610                 else
3611                         val |= DISP_FBC_WM_DIS;
3612                 I915_WRITE(DISP_ARB_CTL, val);
3613         }
3614
3615         if (dirty & WM_DIRTY_LP(1) &&
3616             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3617                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3618
3619         if (INTEL_GEN(dev_priv) >= 7) {
3620                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3621                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3622                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3623                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3624         }
3625
3626         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3627                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3628         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3629                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3630         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3631                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3632
3633         dev_priv->wm.hw = *results;
3634 }
3635
3636 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3637 {
3638         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3639 }
3640
3641 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3642 {
3643         int i;
3644         int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3645         u8 enabled_slices_mask = 0;
3646
3647         for (i = 0; i < max_slices; i++) {
3648                 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3649                         enabled_slices_mask |= BIT(i);
3650         }
3651
3652         return enabled_slices_mask;
3653 }
3654
3655 /*
3656  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3657  * so assume we'll always need it in order to avoid underruns.
3658  */
3659 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3660 {
3661         return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3662 }
3663
3664 static bool
3665 intel_has_sagv(struct drm_i915_private *dev_priv)
3666 {
3667         return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3668                 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3669 }
3670
3671 static void
3672 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3673 {
3674         if (INTEL_GEN(dev_priv) >= 12) {
3675                 u32 val = 0;
3676                 int ret;
3677
3678                 ret = sandybridge_pcode_read(dev_priv,
3679                                              GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3680                                              &val, NULL);
3681                 if (!ret) {
3682                         dev_priv->sagv_block_time_us = val;
3683                         return;
3684                 }
3685
3686                 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3687         } else if (IS_GEN(dev_priv, 11)) {
3688                 dev_priv->sagv_block_time_us = 10;
3689                 return;
3690         } else if (IS_GEN(dev_priv, 10)) {
3691                 dev_priv->sagv_block_time_us = 20;
3692                 return;
3693         } else if (IS_GEN(dev_priv, 9)) {
3694                 dev_priv->sagv_block_time_us = 30;
3695                 return;
3696         } else {
3697                 MISSING_CASE(INTEL_GEN(dev_priv));
3698         }
3699
3700         /* Default to an unusable block time */
3701         dev_priv->sagv_block_time_us = -1;
3702 }
3703
3704 /*
3705  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3706  * depending on power and performance requirements. The display engine access
3707  * to system memory is blocked during the adjustment time. Because of the
3708  * blocking time, having this enabled can cause full system hangs and/or pipe
3709  * underruns if we don't meet all of the following requirements:
3710  *
3711  *  - <= 1 pipe enabled
3712  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3713  *  - We're not using an interlaced display configuration
3714  */
3715 int
3716 intel_enable_sagv(struct drm_i915_private *dev_priv)
3717 {
3718         int ret;
3719
3720         if (!intel_has_sagv(dev_priv))
3721                 return 0;
3722
3723         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3724                 return 0;
3725
3726         drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3727         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3728                                       GEN9_SAGV_ENABLE);
3729
3730         /* We don't need to wait for SAGV when enabling */
3731
3732         /*
3733          * Some skl systems, pre-release machines in particular,
3734          * don't actually have SAGV.
3735          */
3736         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3737                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3738                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3739                 return 0;
3740         } else if (ret < 0) {
3741                 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3742                 return ret;
3743         }
3744
3745         dev_priv->sagv_status = I915_SAGV_ENABLED;
3746         return 0;
3747 }
3748
3749 int
3750 intel_disable_sagv(struct drm_i915_private *dev_priv)
3751 {
3752         int ret;
3753
3754         if (!intel_has_sagv(dev_priv))
3755                 return 0;
3756
3757         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3758                 return 0;
3759
3760         drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3761         /* bspec says to keep retrying for at least 1 ms */
3762         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3763                                 GEN9_SAGV_DISABLE,
3764                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3765                                 1);
3766         /*
3767          * Some skl systems, pre-release machines in particular,
3768          * don't actually have SAGV.
3769          */
3770         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3771                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3772                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3773                 return 0;
3774         } else if (ret < 0) {
3775                 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3776                 return ret;
3777         }
3778
3779         dev_priv->sagv_status = I915_SAGV_DISABLED;
3780         return 0;
3781 }
3782
3783 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3784 {
3785         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3786         const struct intel_bw_state *new_bw_state;
3787         const struct intel_bw_state *old_bw_state;
3788         u32 new_mask = 0;
3789
3790         /*
3791          * Just return if we can't control SAGV or don't have it.
3792          * This is different from situation when we have SAGV but just can't
3793          * afford it due to DBuf limitation - in case if SAGV is completely
3794          * disabled in a BIOS, we are not even allowed to send a PCode request,
3795          * as it will throw an error. So have to check it here.
3796          */
3797         if (!intel_has_sagv(dev_priv))
3798                 return;
3799
3800         new_bw_state = intel_atomic_get_new_bw_state(state);
3801         if (!new_bw_state)
3802                 return;
3803
3804         if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3805                 intel_disable_sagv(dev_priv);
3806                 return;
3807         }
3808
3809         old_bw_state = intel_atomic_get_old_bw_state(state);
3810         /*
3811          * Nothing to mask
3812          */
3813         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3814                 return;
3815
3816         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3817
3818         /*
3819          * If new mask is zero - means there is nothing to mask,
3820          * we can only unmask, which should be done in unmask.
3821          */
3822         if (!new_mask)
3823                 return;
3824
3825         /*
3826          * Restrict required qgv points before updating the configuration.
3827          * According to BSpec we can't mask and unmask qgv points at the same
3828          * time. Also masking should be done before updating the configuration
3829          * and unmasking afterwards.
3830          */
3831         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3832 }
3833
3834 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3835 {
3836         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3837         const struct intel_bw_state *new_bw_state;
3838         const struct intel_bw_state *old_bw_state;
3839         u32 new_mask = 0;
3840
3841         /*
3842          * Just return if we can't control SAGV or don't have it.
3843          * This is different from situation when we have SAGV but just can't
3844          * afford it due to DBuf limitation - in case if SAGV is completely
3845          * disabled in a BIOS, we are not even allowed to send a PCode request,
3846          * as it will throw an error. So have to check it here.
3847          */
3848         if (!intel_has_sagv(dev_priv))
3849                 return;
3850
3851         new_bw_state = intel_atomic_get_new_bw_state(state);
3852         if (!new_bw_state)
3853                 return;
3854
3855         if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3856                 intel_enable_sagv(dev_priv);
3857                 return;
3858         }
3859
3860         old_bw_state = intel_atomic_get_old_bw_state(state);
3861         /*
3862          * Nothing to unmask
3863          */
3864         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3865                 return;
3866
3867         new_mask = new_bw_state->qgv_points_mask;
3868
3869         /*
3870          * Allow required qgv points after updating the configuration.
3871          * According to BSpec we can't mask and unmask qgv points at the same
3872          * time. Also masking should be done before updating the configuration
3873          * and unmasking afterwards.
3874          */
3875         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3876 }
3877
3878 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3879 {
3880         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3881         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3882         struct intel_plane *plane;
3883         const struct intel_plane_state *plane_state;
3884         int level, latency;
3885
3886         if (!intel_has_sagv(dev_priv))
3887                 return false;
3888
3889         if (!crtc_state->hw.active)
3890                 return true;
3891
3892         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3893                 return false;
3894
3895         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3896                 const struct skl_plane_wm *wm =
3897                         &crtc_state->wm.skl.optimal.planes[plane->id];
3898
3899                 /* Skip this plane if it's not enabled */
3900                 if (!wm->wm[0].plane_en)
3901                         continue;
3902
3903                 /* Find the highest enabled wm level for this plane */
3904                 for (level = ilk_wm_max_level(dev_priv);
3905                      !wm->wm[level].plane_en; --level)
3906                      { }
3907
3908                 latency = dev_priv->wm.skl_latency[level];
3909
3910                 if (skl_needs_memory_bw_wa(dev_priv) &&
3911                     plane_state->uapi.fb->modifier ==
3912                     I915_FORMAT_MOD_X_TILED)
3913                         latency += 15;
3914
3915                 /*
3916                  * If any of the planes on this pipe don't enable wm levels that
3917                  * incur memory latencies higher than sagv_block_time_us we
3918                  * can't enable SAGV.
3919                  */
3920                 if (latency < dev_priv->sagv_block_time_us)
3921                         return false;
3922         }
3923
3924         return true;
3925 }
3926
3927 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3928 {
3929         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3930         enum plane_id plane_id;
3931
3932         if (!crtc_state->hw.active)
3933                 return true;
3934
3935         for_each_plane_id_on_crtc(crtc, plane_id) {
3936                 const struct skl_ddb_entry *plane_alloc =
3937                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
3938                 const struct skl_plane_wm *wm =
3939                         &crtc_state->wm.skl.optimal.planes[plane_id];
3940
3941                 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3942                         return false;
3943         }
3944
3945         return true;
3946 }
3947
3948 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3949 {
3950         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3951         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3952
3953         if (INTEL_GEN(dev_priv) >= 12)
3954                 return tgl_crtc_can_enable_sagv(crtc_state);
3955         else
3956                 return skl_crtc_can_enable_sagv(crtc_state);
3957 }
3958
3959 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3960                            const struct intel_bw_state *bw_state)
3961 {
3962         if (INTEL_GEN(dev_priv) < 11 &&
3963             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3964                 return false;
3965
3966         return bw_state->pipe_sagv_reject == 0;
3967 }
3968
3969 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3970 {
3971         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3972         int ret;
3973         struct intel_crtc *crtc;
3974         struct intel_crtc_state *new_crtc_state;
3975         struct intel_bw_state *new_bw_state = NULL;
3976         const struct intel_bw_state *old_bw_state = NULL;
3977         int i;
3978
3979         for_each_new_intel_crtc_in_state(state, crtc,
3980                                          new_crtc_state, i) {
3981                 new_bw_state = intel_atomic_get_bw_state(state);
3982                 if (IS_ERR(new_bw_state))
3983                         return PTR_ERR(new_bw_state);
3984
3985                 old_bw_state = intel_atomic_get_old_bw_state(state);
3986
3987                 if (intel_crtc_can_enable_sagv(new_crtc_state))
3988                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3989                 else
3990                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3991         }
3992
3993         if (!new_bw_state)
3994                 return 0;
3995
3996         new_bw_state->active_pipes =
3997                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3998
3999         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4000                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4001                 if (ret)
4002                         return ret;
4003         }
4004
4005         for_each_new_intel_crtc_in_state(state, crtc,
4006                                          new_crtc_state, i) {
4007                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4008
4009                 /*
4010                  * We store use_sagv_wm in the crtc state rather than relying on
4011                  * that bw state since we have no convenient way to get at the
4012                  * latter from the plane commit hooks (especially in the legacy
4013                  * cursor case)
4014                  */
4015                 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4016                                        intel_can_enable_sagv(dev_priv, new_bw_state);
4017         }
4018
4019         if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4020             intel_can_enable_sagv(dev_priv, old_bw_state)) {
4021                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4022                 if (ret)
4023                         return ret;
4024         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4025                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4026                 if (ret)
4027                         return ret;
4028         }
4029
4030         return 0;
4031 }
4032
4033 /*
4034  * Calculate initial DBuf slice offset, based on slice size
4035  * and mask(i.e if slice size is 1024 and second slice is enabled
4036  * offset would be 1024)
4037  */
4038 static unsigned int
4039 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4040                                 u32 slice_size,
4041                                 u32 ddb_size)
4042 {
4043         unsigned int offset = 0;
4044
4045         if (!dbuf_slice_mask)
4046                 return 0;
4047
4048         offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4049
4050         WARN_ON(offset >= ddb_size);
4051         return offset;
4052 }
4053
4054 u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4055 {
4056         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4057         drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4058
4059         if (INTEL_GEN(dev_priv) < 11)
4060                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4061
4062         return ddb_size;
4063 }
4064
4065 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4066                             const struct skl_ddb_entry *entry)
4067 {
4068         u32 slice_mask = 0;
4069         u16 ddb_size = intel_get_ddb_size(dev_priv);
4070         u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4071         u16 slice_size = ddb_size / num_supported_slices;
4072         u16 start_slice;
4073         u16 end_slice;
4074
4075         if (!skl_ddb_entry_size(entry))
4076                 return 0;
4077
4078         start_slice = entry->start / slice_size;
4079         end_slice = (entry->end - 1) / slice_size;
4080
4081         /*
4082          * Per plane DDB entry can in a really worst case be on multiple slices
4083          * but single entry is anyway contigious.
4084          */
4085         while (start_slice <= end_slice) {
4086                 slice_mask |= BIT(start_slice);
4087                 start_slice++;
4088         }
4089
4090         return slice_mask;
4091 }
4092
4093 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4094                                   u8 active_pipes);
4095
4096 static int
4097 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4098                                    const struct intel_crtc_state *crtc_state,
4099                                    const u64 total_data_rate,
4100                                    struct skl_ddb_entry *alloc, /* out */
4101                                    int *num_active /* out */)
4102 {
4103         struct drm_atomic_state *state = crtc_state->uapi.state;
4104         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4105         struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4106         const struct intel_crtc *crtc;
4107         u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4108         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4109         struct intel_dbuf_state *new_dbuf_state =
4110                 intel_atomic_get_new_dbuf_state(intel_state);
4111         const struct intel_dbuf_state *old_dbuf_state =
4112                 intel_atomic_get_old_dbuf_state(intel_state);
4113         u8 active_pipes = new_dbuf_state->active_pipes;
4114         u16 ddb_size;
4115         u32 ddb_range_size;
4116         u32 i;
4117         u32 dbuf_slice_mask;
4118         u32 offset;
4119         u32 slice_size;
4120         u32 total_slice_mask;
4121         u32 start, end;
4122         int ret;
4123
4124         *num_active = hweight8(active_pipes);
4125
4126         if (!crtc_state->hw.active) {
4127                 alloc->start = 0;
4128                 alloc->end = 0;
4129                 return 0;
4130         }
4131
4132         ddb_size = intel_get_ddb_size(dev_priv);
4133
4134         slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4135
4136         /*
4137          * If the state doesn't change the active CRTC's or there is no
4138          * modeset request, then there's no need to recalculate;
4139          * the existing pipe allocation limits should remain unchanged.
4140          * Note that we're safe from racing commits since any racing commit
4141          * that changes the active CRTC list or do modeset would need to
4142          * grab _all_ crtc locks, including the one we currently hold.
4143          */
4144         if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes &&
4145             !dev_priv->wm.distrust_bios_wm) {
4146                 /*
4147                  * alloc may be cleared by clear_intel_crtc_state,
4148                  * copy from old state to be sure
4149                  *
4150                  * FIXME get rid of this mess
4151                  */
4152                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4153                 return 0;
4154         }
4155
4156         /*
4157          * Get allowed DBuf slices for correspondent pipe and platform.
4158          */
4159         dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4160
4161         /*
4162          * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4163          * and slice size is 1024, the offset would be 1024
4164          */
4165         offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4166                                                  slice_size, ddb_size);
4167
4168         /*
4169          * Figure out total size of allowed DBuf slices, which is basically
4170          * a number of allowed slices for that pipe multiplied by slice size.
4171          * Inside of this
4172          * range ddb entries are still allocated in proportion to display width.
4173          */
4174         ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4175
4176         /*
4177          * Watermark/ddb requirement highly depends upon width of the
4178          * framebuffer, So instead of allocating DDB equally among pipes
4179          * distribute DDB based on resolution/width of the display.
4180          */
4181         total_slice_mask = dbuf_slice_mask;
4182         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4183                 const struct drm_display_mode *adjusted_mode =
4184                         &crtc_state->hw.adjusted_mode;
4185                 enum pipe pipe = crtc->pipe;
4186                 int hdisplay, vdisplay;
4187                 u32 pipe_dbuf_slice_mask;
4188
4189                 if (!crtc_state->hw.active)
4190                         continue;
4191
4192                 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4193                                                                active_pipes);
4194
4195                 /*
4196                  * According to BSpec pipe can share one dbuf slice with another
4197                  * pipes or pipe can use multiple dbufs, in both cases we
4198                  * account for other pipes only if they have exactly same mask.
4199                  * However we need to account how many slices we should enable
4200                  * in total.
4201                  */
4202                 total_slice_mask |= pipe_dbuf_slice_mask;
4203
4204                 /*
4205                  * Do not account pipes using other slice sets
4206                  * luckily as of current BSpec slice sets do not partially
4207                  * intersect(pipes share either same one slice or same slice set
4208                  * i.e no partial intersection), so it is enough to check for
4209                  * equality for now.
4210                  */
4211                 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4212                         continue;
4213
4214                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4215
4216                 total_width_in_range += hdisplay;
4217
4218                 if (pipe < for_pipe)
4219                         width_before_pipe_in_range += hdisplay;
4220                 else if (pipe == for_pipe)
4221                         pipe_width = hdisplay;
4222         }
4223
4224         /*
4225          * FIXME: For now we always enable slice S1 as per
4226          * the Bspec display initialization sequence.
4227          */
4228         new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1);
4229
4230         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4231                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4232                 if (ret)
4233                         return ret;
4234         }
4235
4236         start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4237         end = ddb_range_size *
4238                 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4239
4240         alloc->start = offset + start;
4241         alloc->end = offset + end;
4242
4243         drm_dbg_kms(&dev_priv->drm,
4244                     "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
4245                     for_crtc->base.id, for_crtc->name,
4246                     dbuf_slice_mask, alloc->start, alloc->end, active_pipes);
4247
4248         return 0;
4249 }
4250
4251 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4252                                  int width, const struct drm_format_info *format,
4253                                  u64 modifier, unsigned int rotation,
4254                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
4255                                  int color_plane);
4256 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4257                                  int level,
4258                                  unsigned int latency,
4259                                  const struct skl_wm_params *wp,
4260                                  const struct skl_wm_level *result_prev,
4261                                  struct skl_wm_level *result /* out */);
4262
4263 static unsigned int
4264 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4265                       int num_active)
4266 {
4267         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4268         int level, max_level = ilk_wm_max_level(dev_priv);
4269         struct skl_wm_level wm = {};
4270         int ret, min_ddb_alloc = 0;
4271         struct skl_wm_params wp;
4272
4273         ret = skl_compute_wm_params(crtc_state, 256,
4274                                     drm_format_info(DRM_FORMAT_ARGB8888),
4275                                     DRM_FORMAT_MOD_LINEAR,
4276                                     DRM_MODE_ROTATE_0,
4277                                     crtc_state->pixel_rate, &wp, 0);
4278         drm_WARN_ON(&dev_priv->drm, ret);
4279
4280         for (level = 0; level <= max_level; level++) {
4281                 unsigned int latency = dev_priv->wm.skl_latency[level];
4282
4283                 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4284                 if (wm.min_ddb_alloc == U16_MAX)
4285                         break;
4286
4287                 min_ddb_alloc = wm.min_ddb_alloc;
4288         }
4289
4290         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4291 }
4292
4293 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4294                                        struct skl_ddb_entry *entry, u32 reg)
4295 {
4296
4297         entry->start = reg & DDB_ENTRY_MASK;
4298         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4299
4300         if (entry->end)
4301                 entry->end += 1;
4302 }
4303
4304 static void
4305 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4306                            const enum pipe pipe,
4307                            const enum plane_id plane_id,
4308                            struct skl_ddb_entry *ddb_y,
4309                            struct skl_ddb_entry *ddb_uv)
4310 {
4311         u32 val, val2;
4312         u32 fourcc = 0;
4313
4314         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4315         if (plane_id == PLANE_CURSOR) {
4316                 val = I915_READ(CUR_BUF_CFG(pipe));
4317                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4318                 return;
4319         }
4320
4321         val = I915_READ(PLANE_CTL(pipe, plane_id));
4322
4323         /* No DDB allocated for disabled planes */
4324         if (val & PLANE_CTL_ENABLE)
4325                 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4326                                               val & PLANE_CTL_ORDER_RGBX,
4327                                               val & PLANE_CTL_ALPHA_MASK);
4328
4329         if (INTEL_GEN(dev_priv) >= 11) {
4330                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4331                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4332         } else {
4333                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4334                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4335
4336                 if (fourcc &&
4337                     drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4338                         swap(val, val2);
4339
4340                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4341                 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4342         }
4343 }
4344
4345 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4346                                struct skl_ddb_entry *ddb_y,
4347                                struct skl_ddb_entry *ddb_uv)
4348 {
4349         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4350         enum intel_display_power_domain power_domain;
4351         enum pipe pipe = crtc->pipe;
4352         intel_wakeref_t wakeref;
4353         enum plane_id plane_id;
4354
4355         power_domain = POWER_DOMAIN_PIPE(pipe);
4356         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4357         if (!wakeref)
4358                 return;
4359
4360         for_each_plane_id_on_crtc(crtc, plane_id)
4361                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4362                                            plane_id,
4363                                            &ddb_y[plane_id],
4364                                            &ddb_uv[plane_id]);
4365
4366         intel_display_power_put(dev_priv, power_domain, wakeref);
4367 }
4368
4369 /*
4370  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4371  * The bspec defines downscale amount as:
4372  *
4373  * """
4374  * Horizontal down scale amount = maximum[1, Horizontal source size /
4375  *                                           Horizontal destination size]
4376  * Vertical down scale amount = maximum[1, Vertical source size /
4377  *                                         Vertical destination size]
4378  * Total down scale amount = Horizontal down scale amount *
4379  *                           Vertical down scale amount
4380  * """
4381  *
4382  * Return value is provided in 16.16 fixed point form to retain fractional part.
4383  * Caller should take care of dividing & rounding off the value.
4384  */
4385 static uint_fixed_16_16_t
4386 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4387                            const struct intel_plane_state *plane_state)
4388 {
4389         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4390         u32 src_w, src_h, dst_w, dst_h;
4391         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4392         uint_fixed_16_16_t downscale_h, downscale_w;
4393
4394         if (drm_WARN_ON(&dev_priv->drm,
4395                         !intel_wm_plane_visible(crtc_state, plane_state)))
4396                 return u32_to_fixed16(0);
4397
4398         /*
4399          * Src coordinates are already rotated by 270 degrees for
4400          * the 90/270 degree plane rotation cases (to match the
4401          * GTT mapping), hence no need to account for rotation here.
4402          *
4403          * n.b., src is 16.16 fixed point, dst is whole integer.
4404          */
4405         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4406         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4407         dst_w = drm_rect_width(&plane_state->uapi.dst);
4408         dst_h = drm_rect_height(&plane_state->uapi.dst);
4409
4410         fp_w_ratio = div_fixed16(src_w, dst_w);
4411         fp_h_ratio = div_fixed16(src_h, dst_h);
4412         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4413         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4414
4415         return mul_fixed16(downscale_w, downscale_h);
4416 }
4417
4418 struct dbuf_slice_conf_entry {
4419         u8 active_pipes;
4420         u8 dbuf_mask[I915_MAX_PIPES];
4421 };
4422
4423 /*
4424  * Table taken from Bspec 12716
4425  * Pipes do have some preferred DBuf slice affinity,
4426  * plus there are some hardcoded requirements on how
4427  * those should be distributed for multipipe scenarios.
4428  * For more DBuf slices algorithm can get even more messy
4429  * and less readable, so decided to use a table almost
4430  * as is from BSpec itself - that way it is at least easier
4431  * to compare, change and check.
4432  */
4433 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4434 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4435 {
4436         {
4437                 .active_pipes = BIT(PIPE_A),
4438                 .dbuf_mask = {
4439                         [PIPE_A] = BIT(DBUF_S1),
4440                 },
4441         },
4442         {
4443                 .active_pipes = BIT(PIPE_B),
4444                 .dbuf_mask = {
4445                         [PIPE_B] = BIT(DBUF_S1),
4446                 },
4447         },
4448         {
4449                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4450                 .dbuf_mask = {
4451                         [PIPE_A] = BIT(DBUF_S1),
4452                         [PIPE_B] = BIT(DBUF_S2),
4453                 },
4454         },
4455         {
4456                 .active_pipes = BIT(PIPE_C),
4457                 .dbuf_mask = {
4458                         [PIPE_C] = BIT(DBUF_S2),
4459                 },
4460         },
4461         {
4462                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4463                 .dbuf_mask = {
4464                         [PIPE_A] = BIT(DBUF_S1),
4465                         [PIPE_C] = BIT(DBUF_S2),
4466                 },
4467         },
4468         {
4469                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4470                 .dbuf_mask = {
4471                         [PIPE_B] = BIT(DBUF_S1),
4472                         [PIPE_C] = BIT(DBUF_S2),
4473                 },
4474         },
4475         {
4476                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4477                 .dbuf_mask = {
4478                         [PIPE_A] = BIT(DBUF_S1),
4479                         [PIPE_B] = BIT(DBUF_S1),
4480                         [PIPE_C] = BIT(DBUF_S2),
4481                 },
4482         },
4483         {}
4484 };
4485
4486 /*
4487  * Table taken from Bspec 49255
4488  * Pipes do have some preferred DBuf slice affinity,
4489  * plus there are some hardcoded requirements on how
4490  * those should be distributed for multipipe scenarios.
4491  * For more DBuf slices algorithm can get even more messy
4492  * and less readable, so decided to use a table almost
4493  * as is from BSpec itself - that way it is at least easier
4494  * to compare, change and check.
4495  */
4496 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4497 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4498 {
4499         {
4500                 .active_pipes = BIT(PIPE_A),
4501                 .dbuf_mask = {
4502                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4503                 },
4504         },
4505         {
4506                 .active_pipes = BIT(PIPE_B),
4507                 .dbuf_mask = {
4508                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4509                 },
4510         },
4511         {
4512                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4513                 .dbuf_mask = {
4514                         [PIPE_A] = BIT(DBUF_S2),
4515                         [PIPE_B] = BIT(DBUF_S1),
4516                 },
4517         },
4518         {
4519                 .active_pipes = BIT(PIPE_C),
4520                 .dbuf_mask = {
4521                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4522                 },
4523         },
4524         {
4525                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4526                 .dbuf_mask = {
4527                         [PIPE_A] = BIT(DBUF_S1),
4528                         [PIPE_C] = BIT(DBUF_S2),
4529                 },
4530         },
4531         {
4532                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4533                 .dbuf_mask = {
4534                         [PIPE_B] = BIT(DBUF_S1),
4535                         [PIPE_C] = BIT(DBUF_S2),
4536                 },
4537         },
4538         {
4539                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4540                 .dbuf_mask = {
4541                         [PIPE_A] = BIT(DBUF_S1),
4542                         [PIPE_B] = BIT(DBUF_S1),
4543                         [PIPE_C] = BIT(DBUF_S2),
4544                 },
4545         },
4546         {
4547                 .active_pipes = BIT(PIPE_D),
4548                 .dbuf_mask = {
4549                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4550                 },
4551         },
4552         {
4553                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4554                 .dbuf_mask = {
4555                         [PIPE_A] = BIT(DBUF_S1),
4556                         [PIPE_D] = BIT(DBUF_S2),
4557                 },
4558         },
4559         {
4560                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4561                 .dbuf_mask = {
4562                         [PIPE_B] = BIT(DBUF_S1),
4563                         [PIPE_D] = BIT(DBUF_S2),
4564                 },
4565         },
4566         {
4567                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4568                 .dbuf_mask = {
4569                         [PIPE_A] = BIT(DBUF_S1),
4570                         [PIPE_B] = BIT(DBUF_S1),
4571                         [PIPE_D] = BIT(DBUF_S2),
4572                 },
4573         },
4574         {
4575                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4576                 .dbuf_mask = {
4577                         [PIPE_C] = BIT(DBUF_S1),
4578                         [PIPE_D] = BIT(DBUF_S2),
4579                 },
4580         },
4581         {
4582                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4583                 .dbuf_mask = {
4584                         [PIPE_A] = BIT(DBUF_S1),
4585                         [PIPE_C] = BIT(DBUF_S2),
4586                         [PIPE_D] = BIT(DBUF_S2),
4587                 },
4588         },
4589         {
4590                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4591                 .dbuf_mask = {
4592                         [PIPE_B] = BIT(DBUF_S1),
4593                         [PIPE_C] = BIT(DBUF_S2),
4594                         [PIPE_D] = BIT(DBUF_S2),
4595                 },
4596         },
4597         {
4598                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4599                 .dbuf_mask = {
4600                         [PIPE_A] = BIT(DBUF_S1),
4601                         [PIPE_B] = BIT(DBUF_S1),
4602                         [PIPE_C] = BIT(DBUF_S2),
4603                         [PIPE_D] = BIT(DBUF_S2),
4604                 },
4605         },
4606         {}
4607 };
4608
4609 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4610                               const struct dbuf_slice_conf_entry *dbuf_slices)
4611 {
4612         int i;
4613
4614         for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4615                 if (dbuf_slices[i].active_pipes == active_pipes)
4616                         return dbuf_slices[i].dbuf_mask[pipe];
4617         }
4618         return 0;
4619 }
4620
4621 /*
4622  * This function finds an entry with same enabled pipe configuration and
4623  * returns correspondent DBuf slice mask as stated in BSpec for particular
4624  * platform.
4625  */
4626 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4627 {
4628         /*
4629          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4630          * required calculating "pipe ratio" in order to determine
4631          * if one or two slices can be used for single pipe configurations
4632          * as additional constraint to the existing table.
4633          * However based on recent info, it should be not "pipe ratio"
4634          * but rather ratio between pixel_rate and cdclk with additional
4635          * constants, so for now we are using only table until this is
4636          * clarified. Also this is the reason why crtc_state param is
4637          * still here - we will need it once those additional constraints
4638          * pop up.
4639          */
4640         return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4641 }
4642
4643 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4644 {
4645         return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4646 }
4647
4648 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4649                                   u8 active_pipes)
4650 {
4651         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4652         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4653         enum pipe pipe = crtc->pipe;
4654
4655         if (IS_GEN(dev_priv, 12))
4656                 return tgl_compute_dbuf_slices(pipe, active_pipes);
4657         else if (IS_GEN(dev_priv, 11))
4658                 return icl_compute_dbuf_slices(pipe, active_pipes);
4659         /*
4660          * For anything else just return one slice yet.
4661          * Should be extended for other platforms.
4662          */
4663         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4664 }
4665
4666 static u64
4667 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4668                              const struct intel_plane_state *plane_state,
4669                              int color_plane)
4670 {
4671         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4672         const struct drm_framebuffer *fb = plane_state->hw.fb;
4673         u32 data_rate;
4674         u32 width = 0, height = 0;
4675         uint_fixed_16_16_t down_scale_amount;
4676         u64 rate;
4677
4678         if (!plane_state->uapi.visible)
4679                 return 0;
4680
4681         if (plane->id == PLANE_CURSOR)
4682                 return 0;
4683
4684         if (color_plane == 1 &&
4685             !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4686                 return 0;
4687
4688         /*
4689          * Src coordinates are already rotated by 270 degrees for
4690          * the 90/270 degree plane rotation cases (to match the
4691          * GTT mapping), hence no need to account for rotation here.
4692          */
4693         width = drm_rect_width(&plane_state->uapi.src) >> 16;
4694         height = drm_rect_height(&plane_state->uapi.src) >> 16;
4695
4696         /* UV plane does 1/2 pixel sub-sampling */
4697         if (color_plane == 1) {
4698                 width /= 2;
4699                 height /= 2;
4700         }
4701
4702         data_rate = width * height;
4703
4704         down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4705
4706         rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4707
4708         rate *= fb->format->cpp[color_plane];
4709         return rate;
4710 }
4711
4712 static u64
4713 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4714                                  u64 *plane_data_rate,
4715                                  u64 *uv_plane_data_rate)
4716 {
4717         struct intel_plane *plane;
4718         const struct intel_plane_state *plane_state;
4719         u64 total_data_rate = 0;
4720
4721         /* Calculate and cache data rate for each plane */
4722         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4723                 enum plane_id plane_id = plane->id;
4724                 u64 rate;
4725
4726                 /* packed/y */
4727                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4728                 plane_data_rate[plane_id] = rate;
4729                 total_data_rate += rate;
4730
4731                 /* uv-plane */
4732                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4733                 uv_plane_data_rate[plane_id] = rate;
4734                 total_data_rate += rate;
4735         }
4736
4737         return total_data_rate;
4738 }
4739
4740 static u64
4741 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4742                                  u64 *plane_data_rate)
4743 {
4744         struct intel_plane *plane;
4745         const struct intel_plane_state *plane_state;
4746         u64 total_data_rate = 0;
4747
4748         /* Calculate and cache data rate for each plane */
4749         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4750                 enum plane_id plane_id = plane->id;
4751                 u64 rate;
4752
4753                 if (!plane_state->planar_linked_plane) {
4754                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4755                         plane_data_rate[plane_id] = rate;
4756                         total_data_rate += rate;
4757                 } else {
4758                         enum plane_id y_plane_id;
4759
4760                         /*
4761                          * The slave plane might not iterate in
4762                          * intel_atomic_crtc_state_for_each_plane_state(),
4763                          * and needs the master plane state which may be
4764                          * NULL if we try get_new_plane_state(), so we
4765                          * always calculate from the master.
4766                          */
4767                         if (plane_state->planar_slave)
4768                                 continue;
4769
4770                         /* Y plane rate is calculated on the slave */
4771                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4772                         y_plane_id = plane_state->planar_linked_plane->id;
4773                         plane_data_rate[y_plane_id] = rate;
4774                         total_data_rate += rate;
4775
4776                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4777                         plane_data_rate[plane_id] = rate;
4778                         total_data_rate += rate;
4779                 }
4780         }
4781
4782         return total_data_rate;
4783 }
4784
4785 static const struct skl_wm_level *
4786 skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4787                    enum plane_id plane_id,
4788                    int level)
4789 {
4790         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4791         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4792
4793         if (level == 0 && pipe_wm->use_sagv_wm)
4794                 return &wm->sagv_wm0;
4795
4796         return &wm->wm[level];
4797 }
4798
4799 static int
4800 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4801 {
4802         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4803         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4804         struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4805         u16 alloc_size, start = 0;
4806         u16 total[I915_MAX_PLANES] = {};
4807         u16 uv_total[I915_MAX_PLANES] = {};
4808         u64 total_data_rate;
4809         enum plane_id plane_id;
4810         int num_active;
4811         u64 plane_data_rate[I915_MAX_PLANES] = {};
4812         u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4813         u32 blocks;
4814         int level;
4815         int ret;
4816
4817         /* Clear the partitioning for disabled planes. */
4818         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4819         memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4820
4821         if (!crtc_state->hw.active) {
4822                 struct intel_atomic_state *state =
4823                         to_intel_atomic_state(crtc_state->uapi.state);
4824                 struct intel_dbuf_state *new_dbuf_state =
4825                         intel_atomic_get_new_dbuf_state(state);
4826                 const struct intel_dbuf_state *old_dbuf_state =
4827                         intel_atomic_get_old_dbuf_state(state);
4828
4829                 /*
4830                  * FIXME hack to make sure we compute this sensibly when
4831                  * turning off all the pipes. Otherwise we leave it at
4832                  * whatever we had previously, and then runtime PM will
4833                  * mess it up by turning off all but S1. Remove this
4834                  * once the dbuf state computation flow becomes sane.
4835                  */
4836                 if (new_dbuf_state->active_pipes == 0) {
4837                         new_dbuf_state->enabled_slices = BIT(DBUF_S1);
4838
4839                         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4840                                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4841                                 if (ret)
4842                                         return ret;
4843                         }
4844                 }
4845
4846                 alloc->start = alloc->end = 0;
4847                 return 0;
4848         }
4849
4850         if (INTEL_GEN(dev_priv) >= 11)
4851                 total_data_rate =
4852                         icl_get_total_relative_data_rate(crtc_state,
4853                                                          plane_data_rate);
4854         else
4855                 total_data_rate =
4856                         skl_get_total_relative_data_rate(crtc_state,
4857                                                          plane_data_rate,
4858                                                          uv_plane_data_rate);
4859
4860         ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state,
4861                                                  total_data_rate,
4862                                                  alloc, &num_active);
4863         if (ret)
4864                 return ret;
4865
4866         alloc_size = skl_ddb_entry_size(alloc);
4867         if (alloc_size == 0)
4868                 return 0;
4869
4870         /* Allocate fixed number of blocks for cursor. */
4871         total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4872         alloc_size -= total[PLANE_CURSOR];
4873         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4874                 alloc->end - total[PLANE_CURSOR];
4875         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4876
4877         if (total_data_rate == 0)
4878                 return 0;
4879
4880         /*
4881          * Find the highest watermark level for which we can satisfy the block
4882          * requirement of active planes.
4883          */
4884         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4885                 blocks = 0;
4886                 for_each_plane_id_on_crtc(crtc, plane_id) {
4887                         const struct skl_plane_wm *wm =
4888                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4889
4890                         if (plane_id == PLANE_CURSOR) {
4891                                 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4892                                         drm_WARN_ON(&dev_priv->drm,
4893                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
4894                                         blocks = U32_MAX;
4895                                         break;
4896                                 }
4897                                 continue;
4898                         }
4899
4900                         blocks += wm->wm[level].min_ddb_alloc;
4901                         blocks += wm->uv_wm[level].min_ddb_alloc;
4902                 }
4903
4904                 if (blocks <= alloc_size) {
4905                         alloc_size -= blocks;
4906                         break;
4907                 }
4908         }
4909
4910         if (level < 0) {
4911                 drm_dbg_kms(&dev_priv->drm,
4912                             "Requested display configuration exceeds system DDB limitations");
4913                 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4914                             blocks, alloc_size);
4915                 return -EINVAL;
4916         }
4917
4918         /*
4919          * Grant each plane the blocks it requires at the highest achievable
4920          * watermark level, plus an extra share of the leftover blocks
4921          * proportional to its relative data rate.
4922          */
4923         for_each_plane_id_on_crtc(crtc, plane_id) {
4924                 const struct skl_plane_wm *wm =
4925                         &crtc_state->wm.skl.optimal.planes[plane_id];
4926                 u64 rate;
4927                 u16 extra;
4928
4929                 if (plane_id == PLANE_CURSOR)
4930                         continue;
4931
4932                 /*
4933                  * We've accounted for all active planes; remaining planes are
4934                  * all disabled.
4935                  */
4936                 if (total_data_rate == 0)
4937                         break;
4938
4939                 rate = plane_data_rate[plane_id];
4940                 extra = min_t(u16, alloc_size,
4941                               DIV64_U64_ROUND_UP(alloc_size * rate,
4942                                                  total_data_rate));
4943                 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4944                 alloc_size -= extra;
4945                 total_data_rate -= rate;
4946
4947                 if (total_data_rate == 0)
4948                         break;
4949
4950                 rate = uv_plane_data_rate[plane_id];
4951                 extra = min_t(u16, alloc_size,
4952                               DIV64_U64_ROUND_UP(alloc_size * rate,
4953                                                  total_data_rate));
4954                 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4955                 alloc_size -= extra;
4956                 total_data_rate -= rate;
4957         }
4958         drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4959
4960         /* Set the actual DDB start/end points for each plane */
4961         start = alloc->start;
4962         for_each_plane_id_on_crtc(crtc, plane_id) {
4963                 struct skl_ddb_entry *plane_alloc =
4964                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
4965                 struct skl_ddb_entry *uv_plane_alloc =
4966                         &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4967
4968                 if (plane_id == PLANE_CURSOR)
4969                         continue;
4970
4971                 /* Gen11+ uses a separate plane for UV watermarks */
4972                 drm_WARN_ON(&dev_priv->drm,
4973                             INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4974
4975                 /* Leave disabled planes at (0,0) */
4976                 if (total[plane_id]) {
4977                         plane_alloc->start = start;
4978                         start += total[plane_id];
4979                         plane_alloc->end = start;
4980                 }
4981
4982                 if (uv_total[plane_id]) {
4983                         uv_plane_alloc->start = start;
4984                         start += uv_total[plane_id];
4985                         uv_plane_alloc->end = start;
4986                 }
4987         }
4988
4989         /*
4990          * When we calculated watermark values we didn't know how high
4991          * of a level we'd actually be able to hit, so we just marked
4992          * all levels as "enabled."  Go back now and disable the ones
4993          * that aren't actually possible.
4994          */
4995         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4996                 for_each_plane_id_on_crtc(crtc, plane_id) {
4997                         struct skl_plane_wm *wm =
4998                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4999
5000                         /*
5001                          * We only disable the watermarks for each plane if
5002                          * they exceed the ddb allocation of said plane. This
5003                          * is done so that we don't end up touching cursor
5004                          * watermarks needlessly when some other plane reduces
5005                          * our max possible watermark level.
5006                          *
5007                          * Bspec has this to say about the PLANE_WM enable bit:
5008                          * "All the watermarks at this level for all enabled
5009                          *  planes must be enabled before the level will be used."
5010                          * So this is actually safe to do.
5011                          */
5012                         if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
5013                             wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
5014                                 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
5015
5016                         /*
5017                          * Wa_1408961008:icl, ehl
5018                          * Underruns with WM1+ disabled
5019                          */
5020                         if (IS_GEN(dev_priv, 11) &&
5021                             level == 1 && wm->wm[0].plane_en) {
5022                                 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
5023                                 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
5024                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5025                         }
5026                 }
5027         }
5028
5029         /*
5030          * Go back and disable the transition watermark if it turns out we
5031          * don't have enough DDB blocks for it.
5032          */
5033         for_each_plane_id_on_crtc(crtc, plane_id) {
5034                 struct skl_plane_wm *wm =
5035                         &crtc_state->wm.skl.optimal.planes[plane_id];
5036
5037                 if (wm->trans_wm.plane_res_b >= total[plane_id])
5038                         memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
5039         }
5040
5041         return 0;
5042 }
5043
5044 /*
5045  * The max latency should be 257 (max the punit can code is 255 and we add 2us
5046  * for the read latency) and cpp should always be <= 8, so that
5047  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5048  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5049 */
5050 static uint_fixed_16_16_t
5051 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5052                u8 cpp, u32 latency, u32 dbuf_block_size)
5053 {
5054         u32 wm_intermediate_val;
5055         uint_fixed_16_16_t ret;
5056
5057         if (latency == 0)
5058                 return FP_16_16_MAX;
5059
5060         wm_intermediate_val = latency * pixel_rate * cpp;
5061         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5062
5063         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5064                 ret = add_fixed16_u32(ret, 1);
5065
5066         return ret;
5067 }
5068
5069 static uint_fixed_16_16_t
5070 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5071                uint_fixed_16_16_t plane_blocks_per_line)
5072 {
5073         u32 wm_intermediate_val;
5074         uint_fixed_16_16_t ret;
5075
5076         if (latency == 0)
5077                 return FP_16_16_MAX;
5078
5079         wm_intermediate_val = latency * pixel_rate;
5080         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5081                                            pipe_htotal * 1000);
5082         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5083         return ret;
5084 }
5085
5086 static uint_fixed_16_16_t
5087 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5088 {
5089         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5090         u32 pixel_rate;
5091         u32 crtc_htotal;
5092         uint_fixed_16_16_t linetime_us;
5093
5094         if (!crtc_state->hw.active)
5095                 return u32_to_fixed16(0);
5096
5097         pixel_rate = crtc_state->pixel_rate;
5098
5099         if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5100                 return u32_to_fixed16(0);
5101
5102         crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5103         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5104
5105         return linetime_us;
5106 }
5107
5108 static u32
5109 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5110                               const struct intel_plane_state *plane_state)
5111 {
5112         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5113         u64 adjusted_pixel_rate;
5114         uint_fixed_16_16_t downscale_amount;
5115
5116         /* Shouldn't reach here on disabled planes... */
5117         if (drm_WARN_ON(&dev_priv->drm,
5118                         !intel_wm_plane_visible(crtc_state, plane_state)))
5119                 return 0;
5120
5121         /*
5122          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5123          * with additional adjustments for plane-specific scaling.
5124          */
5125         adjusted_pixel_rate = crtc_state->pixel_rate;
5126         downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5127
5128         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5129                                             downscale_amount);
5130 }
5131
5132 static int
5133 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5134                       int width, const struct drm_format_info *format,
5135                       u64 modifier, unsigned int rotation,
5136                       u32 plane_pixel_rate, struct skl_wm_params *wp,
5137                       int color_plane)
5138 {
5139         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5140         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5141         u32 interm_pbpl;
5142
5143         /* only planar format has two planes */
5144         if (color_plane == 1 &&
5145             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5146                 drm_dbg_kms(&dev_priv->drm,
5147                             "Non planar format have single plane\n");
5148                 return -EINVAL;
5149         }
5150
5151         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5152                       modifier == I915_FORMAT_MOD_Yf_TILED ||
5153                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5154                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5155         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5156         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5157                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5158         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5159
5160         wp->width = width;
5161         if (color_plane == 1 && wp->is_planar)
5162                 wp->width /= 2;
5163
5164         wp->cpp = format->cpp[color_plane];
5165         wp->plane_pixel_rate = plane_pixel_rate;
5166
5167         if (INTEL_GEN(dev_priv) >= 11 &&
5168             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5169                 wp->dbuf_block_size = 256;
5170         else
5171                 wp->dbuf_block_size = 512;
5172
5173         if (drm_rotation_90_or_270(rotation)) {
5174                 switch (wp->cpp) {
5175                 case 1:
5176                         wp->y_min_scanlines = 16;
5177                         break;
5178                 case 2:
5179                         wp->y_min_scanlines = 8;
5180                         break;
5181                 case 4:
5182                         wp->y_min_scanlines = 4;
5183                         break;
5184                 default:
5185                         MISSING_CASE(wp->cpp);
5186                         return -EINVAL;
5187                 }
5188         } else {
5189                 wp->y_min_scanlines = 4;
5190         }
5191
5192         if (skl_needs_memory_bw_wa(dev_priv))
5193                 wp->y_min_scanlines *= 2;
5194
5195         wp->plane_bytes_per_line = wp->width * wp->cpp;
5196         if (wp->y_tiled) {
5197                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5198                                            wp->y_min_scanlines,
5199                                            wp->dbuf_block_size);
5200
5201                 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5202                         interm_pbpl++;
5203
5204                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5205                                                         wp->y_min_scanlines);
5206         } else {
5207                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5208                                            wp->dbuf_block_size);
5209
5210                 if (!wp->x_tiled ||
5211                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5212                         interm_pbpl++;
5213
5214                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5215         }
5216
5217         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5218                                              wp->plane_blocks_per_line);
5219
5220         wp->linetime_us = fixed16_to_u32_round_up(
5221                                         intel_get_linetime_us(crtc_state));
5222
5223         return 0;
5224 }
5225
5226 static int
5227 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5228                             const struct intel_plane_state *plane_state,
5229                             struct skl_wm_params *wp, int color_plane)
5230 {
5231         const struct drm_framebuffer *fb = plane_state->hw.fb;
5232         int width;
5233
5234         /*
5235          * Src coordinates are already rotated by 270 degrees for
5236          * the 90/270 degree plane rotation cases (to match the
5237          * GTT mapping), hence no need to account for rotation here.
5238          */
5239         width = drm_rect_width(&plane_state->uapi.src) >> 16;
5240
5241         return skl_compute_wm_params(crtc_state, width,
5242                                      fb->format, fb->modifier,
5243                                      plane_state->hw.rotation,
5244                                      skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5245                                      wp, color_plane);
5246 }
5247
5248 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5249 {
5250         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5251                 return true;
5252
5253         /* The number of lines are ignored for the level 0 watermark. */
5254         return level > 0;
5255 }
5256
5257 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5258                                  int level,
5259                                  unsigned int latency,
5260                                  const struct skl_wm_params *wp,
5261                                  const struct skl_wm_level *result_prev,
5262                                  struct skl_wm_level *result /* out */)
5263 {
5264         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5265         uint_fixed_16_16_t method1, method2;
5266         uint_fixed_16_16_t selected_result;
5267         u32 res_blocks, res_lines, min_ddb_alloc = 0;
5268
5269         if (latency == 0) {
5270                 /* reject it */
5271                 result->min_ddb_alloc = U16_MAX;
5272                 return;
5273         }
5274
5275         /*
5276          * WaIncreaseLatencyIPCEnabled: kbl,cfl
5277          * Display WA #1141: kbl,cfl
5278          */
5279         if ((IS_KABYLAKE(dev_priv) ||
5280              IS_COFFEELAKE(dev_priv) ||
5281              IS_COMETLAKE(dev_priv)) &&
5282             dev_priv->ipc_enabled)
5283                 latency += 4;
5284
5285         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5286                 latency += 15;
5287
5288         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5289                                  wp->cpp, latency, wp->dbuf_block_size);
5290         method2 = skl_wm_method2(wp->plane_pixel_rate,
5291                                  crtc_state->hw.adjusted_mode.crtc_htotal,
5292                                  latency,
5293                                  wp->plane_blocks_per_line);
5294
5295         if (wp->y_tiled) {
5296                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5297         } else {
5298                 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5299                      wp->dbuf_block_size < 1) &&
5300                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5301                         selected_result = method2;
5302                 } else if (latency >= wp->linetime_us) {
5303                         if (IS_GEN(dev_priv, 9) &&
5304                             !IS_GEMINILAKE(dev_priv))
5305                                 selected_result = min_fixed16(method1, method2);
5306                         else
5307                                 selected_result = method2;
5308                 } else {
5309                         selected_result = method1;
5310                 }
5311         }
5312
5313         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5314         res_lines = div_round_up_fixed16(selected_result,
5315                                          wp->plane_blocks_per_line);
5316
5317         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5318                 /* Display WA #1125: skl,bxt,kbl */
5319                 if (level == 0 && wp->rc_surface)
5320                         res_blocks +=
5321                                 fixed16_to_u32_round_up(wp->y_tile_minimum);
5322
5323                 /* Display WA #1126: skl,bxt,kbl */
5324                 if (level >= 1 && level <= 7) {
5325                         if (wp->y_tiled) {
5326                                 res_blocks +=
5327                                     fixed16_to_u32_round_up(wp->y_tile_minimum);
5328                                 res_lines += wp->y_min_scanlines;
5329                         } else {
5330                                 res_blocks++;
5331                         }
5332
5333                         /*
5334                          * Make sure result blocks for higher latency levels are
5335                          * atleast as high as level below the current level.
5336                          * Assumption in DDB algorithm optimization for special
5337                          * cases. Also covers Display WA #1125 for RC.
5338                          */
5339                         if (result_prev->plane_res_b > res_blocks)
5340                                 res_blocks = result_prev->plane_res_b;
5341                 }
5342         }
5343
5344         if (INTEL_GEN(dev_priv) >= 11) {
5345                 if (wp->y_tiled) {
5346                         int extra_lines;
5347
5348                         if (res_lines % wp->y_min_scanlines == 0)
5349                                 extra_lines = wp->y_min_scanlines;
5350                         else
5351                                 extra_lines = wp->y_min_scanlines * 2 -
5352                                         res_lines % wp->y_min_scanlines;
5353
5354                         min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5355                                                                  wp->plane_blocks_per_line);
5356                 } else {
5357                         min_ddb_alloc = res_blocks +
5358                                 DIV_ROUND_UP(res_blocks, 10);
5359                 }
5360         }
5361
5362         if (!skl_wm_has_lines(dev_priv, level))
5363                 res_lines = 0;
5364
5365         if (res_lines > 31) {
5366                 /* reject it */
5367                 result->min_ddb_alloc = U16_MAX;
5368                 return;
5369         }
5370
5371         /*
5372          * If res_lines is valid, assume we can use this watermark level
5373          * for now.  We'll come back and disable it after we calculate the
5374          * DDB allocation if it turns out we don't actually have enough
5375          * blocks to satisfy it.
5376          */
5377         result->plane_res_b = res_blocks;
5378         result->plane_res_l = res_lines;
5379         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5380         result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5381         result->plane_en = true;
5382 }
5383
5384 static void
5385 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5386                       const struct skl_wm_params *wm_params,
5387                       struct skl_wm_level *levels)
5388 {
5389         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5390         int level, max_level = ilk_wm_max_level(dev_priv);
5391         struct skl_wm_level *result_prev = &levels[0];
5392
5393         for (level = 0; level <= max_level; level++) {
5394                 struct skl_wm_level *result = &levels[level];
5395                 unsigned int latency = dev_priv->wm.skl_latency[level];
5396
5397                 skl_compute_plane_wm(crtc_state, level, latency,
5398                                      wm_params, result_prev, result);
5399
5400                 result_prev = result;
5401         }
5402 }
5403
5404 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5405                                 const struct skl_wm_params *wm_params,
5406                                 struct skl_plane_wm *plane_wm)
5407 {
5408         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5409         struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5410         struct skl_wm_level *levels = plane_wm->wm;
5411         unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5412
5413         skl_compute_plane_wm(crtc_state, 0, latency,
5414                              wm_params, &levels[0],
5415                              sagv_wm);
5416 }
5417
5418 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5419                                       const struct skl_wm_params *wp,
5420                                       struct skl_plane_wm *wm)
5421 {
5422         struct drm_device *dev = crtc_state->uapi.crtc->dev;
5423         const struct drm_i915_private *dev_priv = to_i915(dev);
5424         u16 trans_min, trans_amount, trans_y_tile_min;
5425         u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5426
5427         /* Transition WM don't make any sense if ipc is disabled */
5428         if (!dev_priv->ipc_enabled)
5429                 return;
5430
5431         /*
5432          * WaDisableTWM:skl,kbl,cfl,bxt
5433          * Transition WM are not recommended by HW team for GEN9
5434          */
5435         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5436                 return;
5437
5438         if (INTEL_GEN(dev_priv) >= 11)
5439                 trans_min = 4;
5440         else
5441                 trans_min = 14;
5442
5443         /* Display WA #1140: glk,cnl */
5444         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5445                 trans_amount = 0;
5446         else
5447                 trans_amount = 10; /* This is configurable amount */
5448
5449         trans_offset_b = trans_min + trans_amount;
5450
5451         /*
5452          * The spec asks for Selected Result Blocks for wm0 (the real value),
5453          * not Result Blocks (the integer value). Pay attention to the capital
5454          * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5455          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5456          * and since we later will have to get the ceiling of the sum in the
5457          * transition watermarks calculation, we can just pretend Selected
5458          * Result Blocks is Result Blocks minus 1 and it should work for the
5459          * current platforms.
5460          */
5461         wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5462
5463         if (wp->y_tiled) {
5464                 trans_y_tile_min =
5465                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5466                 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5467                                 trans_offset_b;
5468         } else {
5469                 res_blocks = wm0_sel_res_b + trans_offset_b;
5470         }
5471
5472         /*
5473          * Just assume we can enable the transition watermark.  After
5474          * computing the DDB we'll come back and disable it if that
5475          * assumption turns out to be false.
5476          */
5477         wm->trans_wm.plane_res_b = res_blocks + 1;
5478         wm->trans_wm.plane_en = true;
5479 }
5480
5481 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5482                                      const struct intel_plane_state *plane_state,
5483                                      enum plane_id plane_id, int color_plane)
5484 {
5485         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5486         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5487         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5488         struct skl_wm_params wm_params;
5489         int ret;
5490
5491         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5492                                           &wm_params, color_plane);
5493         if (ret)
5494                 return ret;
5495
5496         skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5497
5498         if (INTEL_GEN(dev_priv) >= 12)
5499                 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5500
5501         skl_compute_transition_wm(crtc_state, &wm_params, wm);
5502
5503         return 0;
5504 }
5505
5506 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5507                                  const struct intel_plane_state *plane_state,
5508                                  enum plane_id plane_id)
5509 {
5510         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5511         struct skl_wm_params wm_params;
5512         int ret;
5513
5514         wm->is_planar = true;
5515
5516         /* uv plane watermarks must also be validated for NV12/Planar */
5517         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5518                                           &wm_params, 1);
5519         if (ret)
5520                 return ret;
5521
5522         skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5523
5524         return 0;
5525 }
5526
5527 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5528                               const struct intel_plane_state *plane_state)
5529 {
5530         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5531         const struct drm_framebuffer *fb = plane_state->hw.fb;
5532         enum plane_id plane_id = plane->id;
5533         int ret;
5534
5535         if (!intel_wm_plane_visible(crtc_state, plane_state))
5536                 return 0;
5537
5538         ret = skl_build_plane_wm_single(crtc_state, plane_state,
5539                                         plane_id, 0);
5540         if (ret)
5541                 return ret;
5542
5543         if (fb->format->is_yuv && fb->format->num_planes > 1) {
5544                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5545                                             plane_id);
5546                 if (ret)
5547                         return ret;
5548         }
5549
5550         return 0;
5551 }
5552
5553 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5554                               const struct intel_plane_state *plane_state)
5555 {
5556         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5557         enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5558         int ret;
5559
5560         /* Watermarks calculated in master */
5561         if (plane_state->planar_slave)
5562                 return 0;
5563
5564         if (plane_state->planar_linked_plane) {
5565                 const struct drm_framebuffer *fb = plane_state->hw.fb;
5566                 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5567
5568                 drm_WARN_ON(&dev_priv->drm,
5569                             !intel_wm_plane_visible(crtc_state, plane_state));
5570                 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5571                             fb->format->num_planes == 1);
5572
5573                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5574                                                 y_plane_id, 0);
5575                 if (ret)
5576                         return ret;
5577
5578                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5579                                                 plane_id, 1);
5580                 if (ret)
5581                         return ret;
5582         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5583                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5584                                                 plane_id, 0);
5585                 if (ret)
5586                         return ret;
5587         }
5588
5589         return 0;
5590 }
5591
5592 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5593 {
5594         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5595         struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5596         struct intel_plane *plane;
5597         const struct intel_plane_state *plane_state;
5598         int ret;
5599
5600         /*
5601          * We'll only calculate watermarks for planes that are actually
5602          * enabled, so make sure all other planes are set as disabled.
5603          */
5604         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5605
5606         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5607                                                      crtc_state) {
5608
5609                 if (INTEL_GEN(dev_priv) >= 11)
5610                         ret = icl_build_plane_wm(crtc_state, plane_state);
5611                 else
5612                         ret = skl_build_plane_wm(crtc_state, plane_state);
5613                 if (ret)
5614                         return ret;
5615         }
5616
5617         return 0;
5618 }
5619
5620 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5621                                 i915_reg_t reg,
5622                                 const struct skl_ddb_entry *entry)
5623 {
5624         if (entry->end)
5625                 intel_de_write_fw(dev_priv, reg,
5626                                   (entry->end - 1) << 16 | entry->start);
5627         else
5628                 intel_de_write_fw(dev_priv, reg, 0);
5629 }
5630
5631 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5632                                i915_reg_t reg,
5633                                const struct skl_wm_level *level)
5634 {
5635         u32 val = 0;
5636
5637         if (level->plane_en)
5638                 val |= PLANE_WM_EN;
5639         if (level->ignore_lines)
5640                 val |= PLANE_WM_IGNORE_LINES;
5641         val |= level->plane_res_b;
5642         val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5643
5644         intel_de_write_fw(dev_priv, reg, val);
5645 }
5646
5647 void skl_write_plane_wm(struct intel_plane *plane,
5648                         const struct intel_crtc_state *crtc_state)
5649 {
5650         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5651         int level, max_level = ilk_wm_max_level(dev_priv);
5652         enum plane_id plane_id = plane->id;
5653         enum pipe pipe = plane->pipe;
5654         const struct skl_plane_wm *wm =
5655                 &crtc_state->wm.skl.optimal.planes[plane_id];
5656         const struct skl_ddb_entry *ddb_y =
5657                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5658         const struct skl_ddb_entry *ddb_uv =
5659                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5660
5661         for (level = 0; level <= max_level; level++) {
5662                 const struct skl_wm_level *wm_level;
5663
5664                 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5665
5666                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5667                                    wm_level);
5668         }
5669         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5670                            &wm->trans_wm);
5671
5672         if (INTEL_GEN(dev_priv) >= 11) {
5673                 skl_ddb_entry_write(dev_priv,
5674                                     PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5675                 return;
5676         }
5677
5678         if (wm->is_planar)
5679                 swap(ddb_y, ddb_uv);
5680
5681         skl_ddb_entry_write(dev_priv,
5682                             PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5683         skl_ddb_entry_write(dev_priv,
5684                             PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5685 }
5686
5687 void skl_write_cursor_wm(struct intel_plane *plane,
5688                          const struct intel_crtc_state *crtc_state)
5689 {
5690         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5691         int level, max_level = ilk_wm_max_level(dev_priv);
5692         enum plane_id plane_id = plane->id;
5693         enum pipe pipe = plane->pipe;
5694         const struct skl_plane_wm *wm =
5695                 &crtc_state->wm.skl.optimal.planes[plane_id];
5696         const struct skl_ddb_entry *ddb =
5697                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5698
5699         for (level = 0; level <= max_level; level++) {
5700                 const struct skl_wm_level *wm_level;
5701
5702                 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5703
5704                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5705                                    wm_level);
5706         }
5707         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5708
5709         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5710 }
5711
5712 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5713                          const struct skl_wm_level *l2)
5714 {
5715         return l1->plane_en == l2->plane_en &&
5716                 l1->ignore_lines == l2->ignore_lines &&
5717                 l1->plane_res_l == l2->plane_res_l &&
5718                 l1->plane_res_b == l2->plane_res_b;
5719 }
5720
5721 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5722                                 const struct skl_plane_wm *wm1,
5723                                 const struct skl_plane_wm *wm2)
5724 {
5725         int level, max_level = ilk_wm_max_level(dev_priv);
5726
5727         for (level = 0; level <= max_level; level++) {
5728                 /*
5729                  * We don't check uv_wm as the hardware doesn't actually
5730                  * use it. It only gets used for calculating the required
5731                  * ddb allocation.
5732                  */
5733                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5734                         return false;
5735         }
5736
5737         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5738 }
5739
5740 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5741                                     const struct skl_ddb_entry *b)
5742 {
5743         return a->start < b->end && b->start < a->end;
5744 }
5745
5746 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5747                                  const struct skl_ddb_entry *entries,
5748                                  int num_entries, int ignore_idx)
5749 {
5750         int i;
5751
5752         for (i = 0; i < num_entries; i++) {
5753                 if (i != ignore_idx &&
5754                     skl_ddb_entries_overlap(ddb, &entries[i]))
5755                         return true;
5756         }
5757
5758         return false;
5759 }
5760
5761 static int
5762 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5763                             struct intel_crtc_state *new_crtc_state)
5764 {
5765         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5766         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5767         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5768         struct intel_plane *plane;
5769
5770         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5771                 struct intel_plane_state *plane_state;
5772                 enum plane_id plane_id = plane->id;
5773
5774                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5775                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5776                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5777                                         &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5778                         continue;
5779
5780                 plane_state = intel_atomic_get_plane_state(state, plane);
5781                 if (IS_ERR(plane_state))
5782                         return PTR_ERR(plane_state);
5783
5784                 new_crtc_state->update_planes |= BIT(plane_id);
5785         }
5786
5787         return 0;
5788 }
5789
5790 static int
5791 skl_compute_ddb(struct intel_atomic_state *state)
5792 {
5793         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5794         const struct intel_dbuf_state *old_dbuf_state;
5795         const struct intel_dbuf_state *new_dbuf_state;
5796         const struct intel_crtc_state *old_crtc_state;
5797         struct intel_crtc_state *new_crtc_state;
5798         struct intel_crtc *crtc;
5799         int ret, i;
5800
5801         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5802                                             new_crtc_state, i) {
5803                 ret = skl_allocate_pipe_ddb(new_crtc_state);
5804                 if (ret)
5805                         return ret;
5806
5807                 ret = skl_ddb_add_affected_planes(old_crtc_state,
5808                                                   new_crtc_state);
5809                 if (ret)
5810                         return ret;
5811         }
5812
5813         old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5814         new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
5815
5816         if (new_dbuf_state &&
5817             new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices)
5818                 drm_dbg_kms(&dev_priv->drm,
5819                             "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5820                             old_dbuf_state->enabled_slices,
5821                             new_dbuf_state->enabled_slices,
5822                             INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5823
5824         return 0;
5825 }
5826
5827 static char enast(bool enable)
5828 {
5829         return enable ? '*' : ' ';
5830 }
5831
5832 static void
5833 skl_print_wm_changes(struct intel_atomic_state *state)
5834 {
5835         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5836         const struct intel_crtc_state *old_crtc_state;
5837         const struct intel_crtc_state *new_crtc_state;
5838         struct intel_plane *plane;
5839         struct intel_crtc *crtc;
5840         int i;
5841
5842         if (!drm_debug_enabled(DRM_UT_KMS))
5843                 return;
5844
5845         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5846                                             new_crtc_state, i) {
5847                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5848
5849                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5850                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5851
5852                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5853                         enum plane_id plane_id = plane->id;
5854                         const struct skl_ddb_entry *old, *new;
5855
5856                         old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5857                         new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5858
5859                         if (skl_ddb_entry_equal(old, new))
5860                                 continue;
5861
5862                         drm_dbg_kms(&dev_priv->drm,
5863                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5864                                     plane->base.base.id, plane->base.name,
5865                                     old->start, old->end, new->start, new->end,
5866                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5867                 }
5868
5869                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5870                         enum plane_id plane_id = plane->id;
5871                         const struct skl_plane_wm *old_wm, *new_wm;
5872
5873                         old_wm = &old_pipe_wm->planes[plane_id];
5874                         new_wm = &new_pipe_wm->planes[plane_id];
5875
5876                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5877                                 continue;
5878
5879                         drm_dbg_kms(&dev_priv->drm,
5880                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5881                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5882                                     plane->base.base.id, plane->base.name,
5883                                     enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5884                                     enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5885                                     enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5886                                     enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5887                                     enast(old_wm->trans_wm.plane_en),
5888                                     enast(old_wm->sagv_wm0.plane_en),
5889                                     enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5890                                     enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5891                                     enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5892                                     enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5893                                     enast(new_wm->trans_wm.plane_en),
5894                                     enast(new_wm->sagv_wm0.plane_en));
5895
5896                         drm_dbg_kms(&dev_priv->drm,
5897                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5898                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5899                                     plane->base.base.id, plane->base.name,
5900                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5901                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5902                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5903                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5904                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5905                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5906                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5907                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5908                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5909                                     enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5910
5911                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5912                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5913                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5914                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5915                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5916                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5917                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5918                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5919                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5920                                     enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5921
5922                         drm_dbg_kms(&dev_priv->drm,
5923                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5924                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5925                                     plane->base.base.id, plane->base.name,
5926                                     old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5927                                     old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5928                                     old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5929                                     old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5930                                     old_wm->trans_wm.plane_res_b,
5931                                     old_wm->sagv_wm0.plane_res_b,
5932                                     new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5933                                     new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5934                                     new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5935                                     new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5936                                     new_wm->trans_wm.plane_res_b,
5937                                     new_wm->sagv_wm0.plane_res_b);
5938
5939                         drm_dbg_kms(&dev_priv->drm,
5940                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5941                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5942                                     plane->base.base.id, plane->base.name,
5943                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5944                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5945                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5946                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5947                                     old_wm->trans_wm.min_ddb_alloc,
5948                                     old_wm->sagv_wm0.min_ddb_alloc,
5949                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5950                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5951                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5952                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5953                                     new_wm->trans_wm.min_ddb_alloc,
5954                                     new_wm->sagv_wm0.min_ddb_alloc);
5955                 }
5956         }
5957 }
5958
5959 static int intel_add_affected_pipes(struct intel_atomic_state *state,
5960                                     u8 pipe_mask)
5961 {
5962         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5963         struct intel_crtc *crtc;
5964
5965         for_each_intel_crtc(&dev_priv->drm, crtc) {
5966                 struct intel_crtc_state *crtc_state;
5967
5968                 if ((pipe_mask & BIT(crtc->pipe)) == 0)
5969                         continue;
5970
5971                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5972                 if (IS_ERR(crtc_state))
5973                         return PTR_ERR(crtc_state);
5974         }
5975
5976         return 0;
5977 }
5978
5979 static int
5980 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5981 {
5982         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5983         struct intel_crtc_state *crtc_state;
5984         struct intel_crtc *crtc;
5985         int i, ret;
5986
5987         if (dev_priv->wm.distrust_bios_wm) {
5988                 /*
5989                  * skl_ddb_get_pipe_allocation_limits() currently requires
5990                  * all active pipes to be included in the state so that
5991                  * it can redistribute the dbuf among them, and it really
5992                  * wants to recompute things when distrust_bios_wm is set
5993                  * so we add all the pipes to the state.
5994                  */
5995                 ret = intel_add_affected_pipes(state, ~0);
5996                 if (ret)
5997                         return ret;
5998         }
5999
6000         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6001                 struct intel_dbuf_state *new_dbuf_state;
6002                 const struct intel_dbuf_state *old_dbuf_state;
6003
6004                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6005                 if (IS_ERR(new_dbuf_state))
6006                         return PTR_ERR(new_dbuf_state);
6007
6008                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6009
6010                 new_dbuf_state->active_pipes =
6011                         intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6012
6013                 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes)
6014                         break;
6015
6016                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6017                 if (ret)
6018                         return ret;
6019
6020                 /*
6021                  * skl_ddb_get_pipe_allocation_limits() currently requires
6022                  * all active pipes to be included in the state so that
6023                  * it can redistribute the dbuf among them.
6024                  */
6025                 ret = intel_add_affected_pipes(state,
6026                                                new_dbuf_state->active_pipes);
6027                 if (ret)
6028                         return ret;
6029
6030                 break;
6031         }
6032
6033         return 0;
6034 }
6035
6036 /*
6037  * To make sure the cursor watermark registers are always consistent
6038  * with our computed state the following scenario needs special
6039  * treatment:
6040  *
6041  * 1. enable cursor
6042  * 2. move cursor entirely offscreen
6043  * 3. disable cursor
6044  *
6045  * Step 2. does call .disable_plane() but does not zero the watermarks
6046  * (since we consider an offscreen cursor still active for the purposes
6047  * of watermarks). Step 3. would not normally call .disable_plane()
6048  * because the actual plane visibility isn't changing, and we don't
6049  * deallocate the cursor ddb until the pipe gets disabled. So we must
6050  * force step 3. to call .disable_plane() to update the watermark
6051  * registers properly.
6052  *
6053  * Other planes do not suffer from this issues as their watermarks are
6054  * calculated based on the actual plane visibility. The only time this
6055  * can trigger for the other planes is during the initial readout as the
6056  * default value of the watermarks registers is not zero.
6057  */
6058 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6059                                       struct intel_crtc *crtc)
6060 {
6061         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6062         const struct intel_crtc_state *old_crtc_state =
6063                 intel_atomic_get_old_crtc_state(state, crtc);
6064         struct intel_crtc_state *new_crtc_state =
6065                 intel_atomic_get_new_crtc_state(state, crtc);
6066         struct intel_plane *plane;
6067
6068         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6069                 struct intel_plane_state *plane_state;
6070                 enum plane_id plane_id = plane->id;
6071
6072                 /*
6073                  * Force a full wm update for every plane on modeset.
6074                  * Required because the reset value of the wm registers
6075                  * is non-zero, whereas we want all disabled planes to
6076                  * have zero watermarks. So if we turn off the relevant
6077                  * power well the hardware state will go out of sync
6078                  * with the software state.
6079                  */
6080                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6081                     skl_plane_wm_equals(dev_priv,
6082                                         &old_crtc_state->wm.skl.optimal.planes[plane_id],
6083                                         &new_crtc_state->wm.skl.optimal.planes[plane_id]))
6084                         continue;
6085
6086                 plane_state = intel_atomic_get_plane_state(state, plane);
6087                 if (IS_ERR(plane_state))
6088                         return PTR_ERR(plane_state);
6089
6090                 new_crtc_state->update_planes |= BIT(plane_id);
6091         }
6092
6093         return 0;
6094 }
6095
6096 static int
6097 skl_compute_wm(struct intel_atomic_state *state)
6098 {
6099         struct intel_crtc *crtc;
6100         struct intel_crtc_state *new_crtc_state;
6101         struct intel_crtc_state *old_crtc_state;
6102         int ret, i;
6103
6104         ret = skl_ddb_add_affected_pipes(state);
6105         if (ret)
6106                 return ret;
6107
6108         /*
6109          * Calculate WM's for all pipes that are part of this transaction.
6110          * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6111          * weren't otherwise being modified if pipe allocations had to change.
6112          */
6113         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6114                                             new_crtc_state, i) {
6115                 ret = skl_build_pipe_wm(new_crtc_state);
6116                 if (ret)
6117                         return ret;
6118         }
6119
6120         ret = skl_compute_ddb(state);
6121         if (ret)
6122                 return ret;
6123
6124         ret = intel_compute_sagv_mask(state);
6125         if (ret)
6126                 return ret;
6127
6128         /*
6129          * skl_compute_ddb() will have adjusted the final watermarks
6130          * based on how much ddb is available. Now we can actually
6131          * check if the final watermarks changed.
6132          */
6133         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6134                                             new_crtc_state, i) {
6135                 ret = skl_wm_add_affected_planes(state, crtc);
6136                 if (ret)
6137                         return ret;
6138         }
6139
6140         skl_print_wm_changes(state);
6141
6142         return 0;
6143 }
6144
6145 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6146                                   struct intel_wm_config *config)
6147 {
6148         struct intel_crtc *crtc;
6149
6150         /* Compute the currently _active_ config */
6151         for_each_intel_crtc(&dev_priv->drm, crtc) {
6152                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6153
6154                 if (!wm->pipe_enabled)
6155                         continue;
6156
6157                 config->sprites_enabled |= wm->sprites_enabled;
6158                 config->sprites_scaled |= wm->sprites_scaled;
6159                 config->num_pipes_active++;
6160         }
6161 }
6162
6163 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6164 {
6165         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6166         struct ilk_wm_maximums max;
6167         struct intel_wm_config config = {};
6168         struct ilk_wm_values results = {};
6169         enum intel_ddb_partitioning partitioning;
6170
6171         ilk_compute_wm_config(dev_priv, &config);
6172
6173         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6174         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6175
6176         /* 5/6 split only in single pipe config on IVB+ */
6177         if (INTEL_GEN(dev_priv) >= 7 &&
6178             config.num_pipes_active == 1 && config.sprites_enabled) {
6179                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6180                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6181
6182                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6183         } else {
6184                 best_lp_wm = &lp_wm_1_2;
6185         }
6186
6187         partitioning = (best_lp_wm == &lp_wm_1_2) ?
6188                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6189
6190         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6191
6192         ilk_write_wm_values(dev_priv, &results);
6193 }
6194
6195 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6196                                    struct intel_crtc *crtc)
6197 {
6198         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6199         const struct intel_crtc_state *crtc_state =
6200                 intel_atomic_get_new_crtc_state(state, crtc);
6201
6202         mutex_lock(&dev_priv->wm.wm_mutex);
6203         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6204         ilk_program_watermarks(dev_priv);
6205         mutex_unlock(&dev_priv->wm.wm_mutex);
6206 }
6207
6208 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6209                                     struct intel_crtc *crtc)
6210 {
6211         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6212         const struct intel_crtc_state *crtc_state =
6213                 intel_atomic_get_new_crtc_state(state, crtc);
6214
6215         if (!crtc_state->wm.need_postvbl_update)
6216                 return;
6217
6218         mutex_lock(&dev_priv->wm.wm_mutex);
6219         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6220         ilk_program_watermarks(dev_priv);
6221         mutex_unlock(&dev_priv->wm.wm_mutex);
6222 }
6223
6224 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6225 {
6226         level->plane_en = val & PLANE_WM_EN;
6227         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6228         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6229         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6230                 PLANE_WM_LINES_MASK;
6231 }
6232
6233 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6234                               struct skl_pipe_wm *out)
6235 {
6236         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6237         enum pipe pipe = crtc->pipe;
6238         int level, max_level;
6239         enum plane_id plane_id;
6240         u32 val;
6241
6242         max_level = ilk_wm_max_level(dev_priv);
6243
6244         for_each_plane_id_on_crtc(crtc, plane_id) {
6245                 struct skl_plane_wm *wm = &out->planes[plane_id];
6246
6247                 for (level = 0; level <= max_level; level++) {
6248                         if (plane_id != PLANE_CURSOR)
6249                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6250                         else
6251                                 val = I915_READ(CUR_WM(pipe, level));
6252
6253                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
6254                 }
6255
6256                 if (INTEL_GEN(dev_priv) >= 12)
6257                         wm->sagv_wm0 = wm->wm[0];
6258
6259                 if (plane_id != PLANE_CURSOR)
6260                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6261                 else
6262                         val = I915_READ(CUR_WM_TRANS(pipe));
6263
6264                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6265         }
6266
6267         if (!crtc->active)
6268                 return;
6269 }
6270
6271 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6272 {
6273         struct intel_crtc *crtc;
6274         struct intel_crtc_state *crtc_state;
6275
6276         for_each_intel_crtc(&dev_priv->drm, crtc) {
6277                 crtc_state = to_intel_crtc_state(crtc->base.state);
6278
6279                 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6280         }
6281
6282         if (dev_priv->active_pipes) {
6283                 /* Fully recompute DDB on first atomic commit */
6284                 dev_priv->wm.distrust_bios_wm = true;
6285         }
6286 }
6287
6288 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6289 {
6290         struct drm_device *dev = crtc->base.dev;
6291         struct drm_i915_private *dev_priv = to_i915(dev);
6292         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6293         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6294         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6295         enum pipe pipe = crtc->pipe;
6296         static const i915_reg_t wm0_pipe_reg[] = {
6297                 [PIPE_A] = WM0_PIPEA_ILK,
6298                 [PIPE_B] = WM0_PIPEB_ILK,
6299                 [PIPE_C] = WM0_PIPEC_IVB,
6300         };
6301
6302         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6303
6304         memset(active, 0, sizeof(*active));
6305
6306         active->pipe_enabled = crtc->active;
6307
6308         if (active->pipe_enabled) {
6309                 u32 tmp = hw->wm_pipe[pipe];
6310
6311                 /*
6312                  * For active pipes LP0 watermark is marked as
6313                  * enabled, and LP1+ watermaks as disabled since
6314                  * we can't really reverse compute them in case
6315                  * multiple pipes are active.
6316                  */
6317                 active->wm[0].enable = true;
6318                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6319                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6320                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6321         } else {
6322                 int level, max_level = ilk_wm_max_level(dev_priv);
6323
6324                 /*
6325                  * For inactive pipes, all watermark levels
6326                  * should be marked as enabled but zeroed,
6327                  * which is what we'd compute them to.
6328                  */
6329                 for (level = 0; level <= max_level; level++)
6330                         active->wm[level].enable = true;
6331         }
6332
6333         crtc->wm.active.ilk = *active;
6334 }
6335
6336 #define _FW_WM(value, plane) \
6337         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6338 #define _FW_WM_VLV(value, plane) \
6339         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6340
6341 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6342                                struct g4x_wm_values *wm)
6343 {
6344         u32 tmp;
6345
6346         tmp = I915_READ(DSPFW1);
6347         wm->sr.plane = _FW_WM(tmp, SR);
6348         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6349         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6350         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6351
6352         tmp = I915_READ(DSPFW2);
6353         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6354         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6355         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6356         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6357         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6358         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6359
6360         tmp = I915_READ(DSPFW3);
6361         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6362         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6363         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6364         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6365 }
6366
6367 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6368                                struct vlv_wm_values *wm)
6369 {
6370         enum pipe pipe;
6371         u32 tmp;
6372
6373         for_each_pipe(dev_priv, pipe) {
6374                 tmp = I915_READ(VLV_DDL(pipe));
6375
6376                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6377                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6378                 wm->ddl[pipe].plane[PLANE_CURSOR] =
6379                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6380                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6381                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6382                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6383                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6384         }
6385
6386         tmp = I915_READ(DSPFW1);
6387         wm->sr.plane = _FW_WM(tmp, SR);
6388         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6389         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6390         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6391
6392         tmp = I915_READ(DSPFW2);
6393         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6394         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6395         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6396
6397         tmp = I915_READ(DSPFW3);
6398         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6399
6400         if (IS_CHERRYVIEW(dev_priv)) {
6401                 tmp = I915_READ(DSPFW7_CHV);
6402                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6403                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6404
6405                 tmp = I915_READ(DSPFW8_CHV);
6406                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6407                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6408
6409                 tmp = I915_READ(DSPFW9_CHV);
6410                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6411                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6412
6413                 tmp = I915_READ(DSPHOWM);
6414                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6415                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6416                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6417                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6418                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6419                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6420                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6421                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6422                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6423                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6424         } else {
6425                 tmp = I915_READ(DSPFW7);
6426                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6427                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6428
6429                 tmp = I915_READ(DSPHOWM);
6430                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6431                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6432                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6433                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6434                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6435                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6436                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6437         }
6438 }
6439
6440 #undef _FW_WM
6441 #undef _FW_WM_VLV
6442
6443 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6444 {
6445         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6446         struct intel_crtc *crtc;
6447
6448         g4x_read_wm_values(dev_priv, wm);
6449
6450         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6451
6452         for_each_intel_crtc(&dev_priv->drm, crtc) {
6453                 struct intel_crtc_state *crtc_state =
6454                         to_intel_crtc_state(crtc->base.state);
6455                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6456                 struct g4x_pipe_wm *raw;
6457                 enum pipe pipe = crtc->pipe;
6458                 enum plane_id plane_id;
6459                 int level, max_level;
6460
6461                 active->cxsr = wm->cxsr;
6462                 active->hpll_en = wm->hpll_en;
6463                 active->fbc_en = wm->fbc_en;
6464
6465                 active->sr = wm->sr;
6466                 active->hpll = wm->hpll;
6467
6468                 for_each_plane_id_on_crtc(crtc, plane_id) {
6469                         active->wm.plane[plane_id] =
6470                                 wm->pipe[pipe].plane[plane_id];
6471                 }
6472
6473                 if (wm->cxsr && wm->hpll_en)
6474                         max_level = G4X_WM_LEVEL_HPLL;
6475                 else if (wm->cxsr)
6476                         max_level = G4X_WM_LEVEL_SR;
6477                 else
6478                         max_level = G4X_WM_LEVEL_NORMAL;
6479
6480                 level = G4X_WM_LEVEL_NORMAL;
6481                 raw = &crtc_state->wm.g4x.raw[level];
6482                 for_each_plane_id_on_crtc(crtc, plane_id)
6483                         raw->plane[plane_id] = active->wm.plane[plane_id];
6484
6485                 if (++level > max_level)
6486                         goto out;
6487
6488                 raw = &crtc_state->wm.g4x.raw[level];
6489                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6490                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6491                 raw->plane[PLANE_SPRITE0] = 0;
6492                 raw->fbc = active->sr.fbc;
6493
6494                 if (++level > max_level)
6495                         goto out;
6496
6497                 raw = &crtc_state->wm.g4x.raw[level];
6498                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6499                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6500                 raw->plane[PLANE_SPRITE0] = 0;
6501                 raw->fbc = active->hpll.fbc;
6502
6503         out:
6504                 for_each_plane_id_on_crtc(crtc, plane_id)
6505                         g4x_raw_plane_wm_set(crtc_state, level,
6506                                              plane_id, USHRT_MAX);
6507                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6508
6509                 crtc_state->wm.g4x.optimal = *active;
6510                 crtc_state->wm.g4x.intermediate = *active;
6511
6512                 drm_dbg_kms(&dev_priv->drm,
6513                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6514                             pipe_name(pipe),
6515                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6516                             wm->pipe[pipe].plane[PLANE_CURSOR],
6517                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
6518         }
6519
6520         drm_dbg_kms(&dev_priv->drm,
6521                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6522                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6523         drm_dbg_kms(&dev_priv->drm,
6524                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6525                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6526         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6527                     yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6528 }
6529
6530 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6531 {
6532         struct intel_plane *plane;
6533         struct intel_crtc *crtc;
6534
6535         mutex_lock(&dev_priv->wm.wm_mutex);
6536
6537         for_each_intel_plane(&dev_priv->drm, plane) {
6538                 struct intel_crtc *crtc =
6539                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6540                 struct intel_crtc_state *crtc_state =
6541                         to_intel_crtc_state(crtc->base.state);
6542                 struct intel_plane_state *plane_state =
6543                         to_intel_plane_state(plane->base.state);
6544                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6545                 enum plane_id plane_id = plane->id;
6546                 int level;
6547
6548                 if (plane_state->uapi.visible)
6549                         continue;
6550
6551                 for (level = 0; level < 3; level++) {
6552                         struct g4x_pipe_wm *raw =
6553                                 &crtc_state->wm.g4x.raw[level];
6554
6555                         raw->plane[plane_id] = 0;
6556                         wm_state->wm.plane[plane_id] = 0;
6557                 }
6558
6559                 if (plane_id == PLANE_PRIMARY) {
6560                         for (level = 0; level < 3; level++) {
6561                                 struct g4x_pipe_wm *raw =
6562                                         &crtc_state->wm.g4x.raw[level];
6563                                 raw->fbc = 0;
6564                         }
6565
6566                         wm_state->sr.fbc = 0;
6567                         wm_state->hpll.fbc = 0;
6568                         wm_state->fbc_en = false;
6569                 }
6570         }
6571
6572         for_each_intel_crtc(&dev_priv->drm, crtc) {
6573                 struct intel_crtc_state *crtc_state =
6574                         to_intel_crtc_state(crtc->base.state);
6575
6576                 crtc_state->wm.g4x.intermediate =
6577                         crtc_state->wm.g4x.optimal;
6578                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6579         }
6580
6581         g4x_program_watermarks(dev_priv);
6582
6583         mutex_unlock(&dev_priv->wm.wm_mutex);
6584 }
6585
6586 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6587 {
6588         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6589         struct intel_crtc *crtc;
6590         u32 val;
6591
6592         vlv_read_wm_values(dev_priv, wm);
6593
6594         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6595         wm->level = VLV_WM_LEVEL_PM2;
6596
6597         if (IS_CHERRYVIEW(dev_priv)) {
6598                 vlv_punit_get(dev_priv);
6599
6600                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6601                 if (val & DSP_MAXFIFO_PM5_ENABLE)
6602                         wm->level = VLV_WM_LEVEL_PM5;
6603
6604                 /*
6605                  * If DDR DVFS is disabled in the BIOS, Punit
6606                  * will never ack the request. So if that happens
6607                  * assume we don't have to enable/disable DDR DVFS
6608                  * dynamically. To test that just set the REQ_ACK
6609                  * bit to poke the Punit, but don't change the
6610                  * HIGH/LOW bits so that we don't actually change
6611                  * the current state.
6612                  */
6613                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6614                 val |= FORCE_DDR_FREQ_REQ_ACK;
6615                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6616
6617                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6618                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6619                         drm_dbg_kms(&dev_priv->drm,
6620                                     "Punit not acking DDR DVFS request, "
6621                                     "assuming DDR DVFS is disabled\n");
6622                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6623                 } else {
6624                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6625                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6626                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6627                 }
6628
6629                 vlv_punit_put(dev_priv);
6630         }
6631
6632         for_each_intel_crtc(&dev_priv->drm, crtc) {
6633                 struct intel_crtc_state *crtc_state =
6634                         to_intel_crtc_state(crtc->base.state);
6635                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6636                 const struct vlv_fifo_state *fifo_state =
6637                         &crtc_state->wm.vlv.fifo_state;
6638                 enum pipe pipe = crtc->pipe;
6639                 enum plane_id plane_id;
6640                 int level;
6641
6642                 vlv_get_fifo_size(crtc_state);
6643
6644                 active->num_levels = wm->level + 1;
6645                 active->cxsr = wm->cxsr;
6646
6647                 for (level = 0; level < active->num_levels; level++) {
6648                         struct g4x_pipe_wm *raw =
6649                                 &crtc_state->wm.vlv.raw[level];
6650
6651                         active->sr[level].plane = wm->sr.plane;
6652                         active->sr[level].cursor = wm->sr.cursor;
6653
6654                         for_each_plane_id_on_crtc(crtc, plane_id) {
6655                                 active->wm[level].plane[plane_id] =
6656                                         wm->pipe[pipe].plane[plane_id];
6657
6658                                 raw->plane[plane_id] =
6659                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
6660                                                             fifo_state->plane[plane_id]);
6661                         }
6662                 }
6663
6664                 for_each_plane_id_on_crtc(crtc, plane_id)
6665                         vlv_raw_plane_wm_set(crtc_state, level,
6666                                              plane_id, USHRT_MAX);
6667                 vlv_invalidate_wms(crtc, active, level);
6668
6669                 crtc_state->wm.vlv.optimal = *active;
6670                 crtc_state->wm.vlv.intermediate = *active;
6671
6672                 drm_dbg_kms(&dev_priv->drm,
6673                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6674                             pipe_name(pipe),
6675                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6676                             wm->pipe[pipe].plane[PLANE_CURSOR],
6677                             wm->pipe[pipe].plane[PLANE_SPRITE0],
6678                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
6679         }
6680
6681         drm_dbg_kms(&dev_priv->drm,
6682                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6683                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6684 }
6685
6686 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6687 {
6688         struct intel_plane *plane;
6689         struct intel_crtc *crtc;
6690
6691         mutex_lock(&dev_priv->wm.wm_mutex);
6692
6693         for_each_intel_plane(&dev_priv->drm, plane) {
6694                 struct intel_crtc *crtc =
6695                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6696                 struct intel_crtc_state *crtc_state =
6697                         to_intel_crtc_state(crtc->base.state);
6698                 struct intel_plane_state *plane_state =
6699                         to_intel_plane_state(plane->base.state);
6700                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6701                 const struct vlv_fifo_state *fifo_state =
6702                         &crtc_state->wm.vlv.fifo_state;
6703                 enum plane_id plane_id = plane->id;
6704                 int level;
6705
6706                 if (plane_state->uapi.visible)
6707                         continue;
6708
6709                 for (level = 0; level < wm_state->num_levels; level++) {
6710                         struct g4x_pipe_wm *raw =
6711                                 &crtc_state->wm.vlv.raw[level];
6712
6713                         raw->plane[plane_id] = 0;
6714
6715                         wm_state->wm[level].plane[plane_id] =
6716                                 vlv_invert_wm_value(raw->plane[plane_id],
6717                                                     fifo_state->plane[plane_id]);
6718                 }
6719         }
6720
6721         for_each_intel_crtc(&dev_priv->drm, crtc) {
6722                 struct intel_crtc_state *crtc_state =
6723                         to_intel_crtc_state(crtc->base.state);
6724
6725                 crtc_state->wm.vlv.intermediate =
6726                         crtc_state->wm.vlv.optimal;
6727                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6728         }
6729
6730         vlv_program_watermarks(dev_priv);
6731
6732         mutex_unlock(&dev_priv->wm.wm_mutex);
6733 }
6734
6735 /*
6736  * FIXME should probably kill this and improve
6737  * the real watermark readout/sanitation instead
6738  */
6739 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6740 {
6741         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6742         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6743         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6744
6745         /*
6746          * Don't touch WM1S_LP_EN here.
6747          * Doing so could cause underruns.
6748          */
6749 }
6750
6751 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6752 {
6753         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6754         struct intel_crtc *crtc;
6755
6756         ilk_init_lp_watermarks(dev_priv);
6757
6758         for_each_intel_crtc(&dev_priv->drm, crtc)
6759                 ilk_pipe_wm_get_hw_state(crtc);
6760
6761         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6762         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6763         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6764
6765         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6766         if (INTEL_GEN(dev_priv) >= 7) {
6767                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6768                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6769         }
6770
6771         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6772                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6773                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6774         else if (IS_IVYBRIDGE(dev_priv))
6775                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6776                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6777
6778         hw->enable_fbc_wm =
6779                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6780 }
6781
6782 /**
6783  * intel_update_watermarks - update FIFO watermark values based on current modes
6784  * @crtc: the #intel_crtc on which to compute the WM
6785  *
6786  * Calculate watermark values for the various WM regs based on current mode
6787  * and plane configuration.
6788  *
6789  * There are several cases to deal with here:
6790  *   - normal (i.e. non-self-refresh)
6791  *   - self-refresh (SR) mode
6792  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6793  *   - lines are small relative to FIFO size (buffer can hold more than 2
6794  *     lines), so need to account for TLB latency
6795  *
6796  *   The normal calculation is:
6797  *     watermark = dotclock * bytes per pixel * latency
6798  *   where latency is platform & configuration dependent (we assume pessimal
6799  *   values here).
6800  *
6801  *   The SR calculation is:
6802  *     watermark = (trunc(latency/line time)+1) * surface width *
6803  *       bytes per pixel
6804  *   where
6805  *     line time = htotal / dotclock
6806  *     surface width = hdisplay for normal plane and 64 for cursor
6807  *   and latency is assumed to be high, as above.
6808  *
6809  * The final value programmed to the register should always be rounded up,
6810  * and include an extra 2 entries to account for clock crossings.
6811  *
6812  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6813  * to set the non-SR watermarks to 8.
6814  */
6815 void intel_update_watermarks(struct intel_crtc *crtc)
6816 {
6817         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6818
6819         if (dev_priv->display.update_wm)
6820                 dev_priv->display.update_wm(crtc);
6821 }
6822
6823 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6824 {
6825         u32 val;
6826
6827         if (!HAS_IPC(dev_priv))
6828                 return;
6829
6830         val = I915_READ(DISP_ARB_CTL2);
6831
6832         if (dev_priv->ipc_enabled)
6833                 val |= DISP_IPC_ENABLE;
6834         else
6835                 val &= ~DISP_IPC_ENABLE;
6836
6837         I915_WRITE(DISP_ARB_CTL2, val);
6838 }
6839
6840 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6841 {
6842         /* Display WA #0477 WaDisableIPC: skl */
6843         if (IS_SKYLAKE(dev_priv))
6844                 return false;
6845
6846         /* Display WA #1141: SKL:all KBL:all CFL */
6847         if (IS_KABYLAKE(dev_priv) ||
6848             IS_COFFEELAKE(dev_priv) ||
6849             IS_COMETLAKE(dev_priv))
6850                 return dev_priv->dram_info.symmetric_memory;
6851
6852         return true;
6853 }
6854
6855 void intel_init_ipc(struct drm_i915_private *dev_priv)
6856 {
6857         if (!HAS_IPC(dev_priv))
6858                 return;
6859
6860         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6861
6862         intel_enable_ipc(dev_priv);
6863 }
6864
6865 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6866 {
6867         /*
6868          * On Ibex Peak and Cougar Point, we need to disable clock
6869          * gating for the panel power sequencer or it will fail to
6870          * start up when no ports are active.
6871          */
6872         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6873 }
6874
6875 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6876 {
6877         enum pipe pipe;
6878
6879         for_each_pipe(dev_priv, pipe) {
6880                 I915_WRITE(DSPCNTR(pipe),
6881                            I915_READ(DSPCNTR(pipe)) |
6882                            DISPPLANE_TRICKLE_FEED_DISABLE);
6883
6884                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6885                 POSTING_READ(DSPSURF(pipe));
6886         }
6887 }
6888
6889 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6890 {
6891         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6892
6893         /*
6894          * Required for FBC
6895          * WaFbcDisableDpfcClockGating:ilk
6896          */
6897         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6898                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6899                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6900
6901         I915_WRITE(PCH_3DCGDIS0,
6902                    MARIUNIT_CLOCK_GATE_DISABLE |
6903                    SVSMUNIT_CLOCK_GATE_DISABLE);
6904         I915_WRITE(PCH_3DCGDIS1,
6905                    VFMUNIT_CLOCK_GATE_DISABLE);
6906
6907         /*
6908          * According to the spec the following bits should be set in
6909          * order to enable memory self-refresh
6910          * The bit 22/21 of 0x42004
6911          * The bit 5 of 0x42020
6912          * The bit 15 of 0x45000
6913          */
6914         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6915                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
6916                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6917         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6918         I915_WRITE(DISP_ARB_CTL,
6919                    (I915_READ(DISP_ARB_CTL) |
6920                     DISP_FBC_WM_DIS));
6921
6922         /*
6923          * Based on the document from hardware guys the following bits
6924          * should be set unconditionally in order to enable FBC.
6925          * The bit 22 of 0x42000
6926          * The bit 22 of 0x42004
6927          * The bit 7,8,9 of 0x42020.
6928          */
6929         if (IS_IRONLAKE_M(dev_priv)) {
6930                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6931                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6932                            I915_READ(ILK_DISPLAY_CHICKEN1) |
6933                            ILK_FBCQ_DIS);
6934                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6935                            I915_READ(ILK_DISPLAY_CHICKEN2) |
6936                            ILK_DPARB_GATE);
6937         }
6938
6939         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6940
6941         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6942                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6943                    ILK_ELPIN_409_SELECT);
6944
6945         g4x_disable_trickle_feed(dev_priv);
6946
6947         ibx_init_clock_gating(dev_priv);
6948 }
6949
6950 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6951 {
6952         enum pipe pipe;
6953         u32 val;
6954
6955         /*
6956          * On Ibex Peak and Cougar Point, we need to disable clock
6957          * gating for the panel power sequencer or it will fail to
6958          * start up when no ports are active.
6959          */
6960         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6961                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6962                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6963         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6964                    DPLS_EDP_PPS_FIX_DIS);
6965         /* The below fixes the weird display corruption, a few pixels shifted
6966          * downward, on (only) LVDS of some HP laptops with IVY.
6967          */
6968         for_each_pipe(dev_priv, pipe) {
6969                 val = I915_READ(TRANS_CHICKEN2(pipe));
6970                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6971                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6972                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6973                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6974                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6975                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6976                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6977         }
6978         /* WADP0ClockGatingDisable */
6979         for_each_pipe(dev_priv, pipe) {
6980                 I915_WRITE(TRANS_CHICKEN1(pipe),
6981                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6982         }
6983 }
6984
6985 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6986 {
6987         u32 tmp;
6988
6989         tmp = I915_READ(MCH_SSKPD);
6990         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6991                 drm_dbg_kms(&dev_priv->drm,
6992                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6993                             tmp);
6994 }
6995
6996 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6997 {
6998         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6999
7000         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
7001
7002         I915_WRITE(ILK_DISPLAY_CHICKEN2,
7003                    I915_READ(ILK_DISPLAY_CHICKEN2) |
7004                    ILK_ELPIN_409_SELECT);
7005
7006         I915_WRITE(GEN6_UCGCTL1,
7007                    I915_READ(GEN6_UCGCTL1) |
7008                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7009                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7010
7011         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7012          * gating disable must be set.  Failure to set it results in
7013          * flickering pixels due to Z write ordering failures after
7014          * some amount of runtime in the Mesa "fire" demo, and Unigine
7015          * Sanctuary and Tropics, and apparently anything else with
7016          * alpha test or pixel discard.
7017          *
7018          * According to the spec, bit 11 (RCCUNIT) must also be set,
7019          * but we didn't debug actual testcases to find it out.
7020          *
7021          * WaDisableRCCUnitClockGating:snb
7022          * WaDisableRCPBUnitClockGating:snb
7023          */
7024         I915_WRITE(GEN6_UCGCTL2,
7025                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7026                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7027
7028         /*
7029          * According to the spec the following bits should be
7030          * set in order to enable memory self-refresh and fbc:
7031          * The bit21 and bit22 of 0x42000
7032          * The bit21 and bit22 of 0x42004
7033          * The bit5 and bit7 of 0x42020
7034          * The bit14 of 0x70180
7035          * The bit14 of 0x71180
7036          *
7037          * WaFbcAsynchFlipDisableFbcQueue:snb
7038          */
7039         I915_WRITE(ILK_DISPLAY_CHICKEN1,
7040                    I915_READ(ILK_DISPLAY_CHICKEN1) |
7041                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7042         I915_WRITE(ILK_DISPLAY_CHICKEN2,
7043                    I915_READ(ILK_DISPLAY_CHICKEN2) |
7044                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7045         I915_WRITE(ILK_DSPCLK_GATE_D,
7046                    I915_READ(ILK_DSPCLK_GATE_D) |
7047                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
7048                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7049
7050         g4x_disable_trickle_feed(dev_priv);
7051
7052         cpt_init_clock_gating(dev_priv);
7053
7054         gen6_check_mch_setup(dev_priv);
7055 }
7056
7057 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7058 {
7059         /*
7060          * TODO: this bit should only be enabled when really needed, then
7061          * disabled when not needed anymore in order to save power.
7062          */
7063         if (HAS_PCH_LPT_LP(dev_priv))
7064                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7065                            I915_READ(SOUTH_DSPCLK_GATE_D) |
7066                            PCH_LP_PARTITION_LEVEL_DISABLE);
7067
7068         /* WADPOClockGatingDisable:hsw */
7069         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7070                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7071                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7072 }
7073
7074 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7075 {
7076         if (HAS_PCH_LPT_LP(dev_priv)) {
7077                 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7078
7079                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7080                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7081         }
7082 }
7083
7084 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7085                                    int general_prio_credits,
7086                                    int high_prio_credits)
7087 {
7088         u32 misccpctl;
7089         u32 val;
7090
7091         /* WaTempDisableDOPClkGating:bdw */
7092         misccpctl = I915_READ(GEN7_MISCCPCTL);
7093         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7094
7095         val = I915_READ(GEN8_L3SQCREG1);
7096         val &= ~L3_PRIO_CREDITS_MASK;
7097         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7098         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7099         I915_WRITE(GEN8_L3SQCREG1, val);
7100
7101         /*
7102          * Wait at least 100 clocks before re-enabling clock gating.
7103          * See the definition of L3SQCREG1 in BSpec.
7104          */
7105         POSTING_READ(GEN8_L3SQCREG1);
7106         udelay(1);
7107         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7108 }
7109
7110 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7111 {
7112         /* Wa_1409120013:icl,ehl */
7113         I915_WRITE(ILK_DPFC_CHICKEN,
7114                    ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7115
7116         /* This is not an Wa. Enable to reduce Sampler power */
7117         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7118                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7119
7120         /*Wa_14010594013:icl, ehl */
7121         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7122                          0, CNL_DELAY_PMRSP);
7123 }
7124
7125 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7126 {
7127         u32 vd_pg_enable = 0;
7128         unsigned int i;
7129
7130         /* Wa_1409120013:tgl */
7131         I915_WRITE(ILK_DPFC_CHICKEN,
7132                    ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7133
7134         /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7135         for (i = 0; i < I915_MAX_VCS; i++) {
7136                 if (HAS_ENGINE(&dev_priv->gt, _VCS(i)))
7137                         vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7138                                         VDN_MFX_POWERGATE_ENABLE(i);
7139         }
7140
7141         I915_WRITE(POWERGATE_ENABLE,
7142                    I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7143
7144         /* Wa_1409825376:tgl (pre-prod)*/
7145         if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
7146                 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7147                            TGL_VRH_GATING_DIS);
7148
7149         /* Wa_14011059788:tgl */
7150         intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7151                          0, DFR_DISABLE);
7152 }
7153
7154 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7155 {
7156         if (!HAS_PCH_CNP(dev_priv))
7157                 return;
7158
7159         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7160         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7161                    CNP_PWM_CGE_GATING_DISABLE);
7162 }
7163
7164 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7165 {
7166         u32 val;
7167         cnp_init_clock_gating(dev_priv);
7168
7169         /* This is not an Wa. Enable for better image quality */
7170         I915_WRITE(_3D_CHICKEN3,
7171                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7172
7173         /* WaEnableChickenDCPR:cnl */
7174         I915_WRITE(GEN8_CHICKEN_DCPR_1,
7175                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7176
7177         /*
7178          * WaFbcWakeMemOn:cnl
7179          * Display WA #0859: cnl
7180          */
7181         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7182                    DISP_FBC_MEMORY_WAKE);
7183
7184         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7185         /* ReadHitWriteOnlyDisable:cnl */
7186         val |= RCCUNIT_CLKGATE_DIS;
7187         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7188
7189         /* Wa_2201832410:cnl */
7190         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7191         val |= GWUNIT_CLKGATE_DIS;
7192         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7193
7194         /* WaDisableVFclkgate:cnl */
7195         /* WaVFUnitClockGatingDisable:cnl */
7196         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7197         val |= VFUNIT_CLKGATE_DIS;
7198         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7199 }
7200
7201 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7202 {
7203         cnp_init_clock_gating(dev_priv);
7204         gen9_init_clock_gating(dev_priv);
7205
7206         /*
7207          * WaFbcTurnOffFbcWatermark:cfl
7208          * Display WA #0562: cfl
7209          */
7210         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7211                    DISP_FBC_WM_DIS);
7212
7213         /*
7214          * WaFbcNukeOnHostModify:cfl
7215          * Display WA #0873: cfl
7216          */
7217         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7218                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7219 }
7220
7221 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7222 {
7223         gen9_init_clock_gating(dev_priv);
7224
7225         /* WaDisableSDEUnitClockGating:kbl */
7226         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7227                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7228                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7229
7230         /* WaDisableGamClockGating:kbl */
7231         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7232                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7233                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7234
7235         /*
7236          * WaFbcTurnOffFbcWatermark:kbl
7237          * Display WA #0562: kbl
7238          */
7239         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7240                    DISP_FBC_WM_DIS);
7241
7242         /*
7243          * WaFbcNukeOnHostModify:kbl
7244          * Display WA #0873: kbl
7245          */
7246         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7247                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7248 }
7249
7250 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7251 {
7252         gen9_init_clock_gating(dev_priv);
7253
7254         /* WAC6entrylatency:skl */
7255         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7256                    FBC_LLC_FULLY_OPEN);
7257
7258         /*
7259          * WaFbcTurnOffFbcWatermark:skl
7260          * Display WA #0562: skl
7261          */
7262         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7263                    DISP_FBC_WM_DIS);
7264
7265         /*
7266          * WaFbcNukeOnHostModify:skl
7267          * Display WA #0873: skl
7268          */
7269         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7270                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7271
7272         /*
7273          * WaFbcHighMemBwCorruptionAvoidance:skl
7274          * Display WA #0883: skl
7275          */
7276         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7277                    ILK_DPFC_DISABLE_DUMMY0);
7278 }
7279
7280 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7281 {
7282         enum pipe pipe;
7283
7284         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7285         I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7286                    I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7287                    HSW_FBCQ_DIS);
7288
7289         /* WaSwitchSolVfFArbitrationPriority:bdw */
7290         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7291
7292         /* WaPsrDPAMaskVBlankInSRD:bdw */
7293         I915_WRITE(CHICKEN_PAR1_1,
7294                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7295
7296         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7297         for_each_pipe(dev_priv, pipe) {
7298                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7299                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
7300                            BDW_DPRS_MASK_VBLANK_SRD);
7301         }
7302
7303         /* WaVSRefCountFullforceMissDisable:bdw */
7304         /* WaDSRefCountFullforceMissDisable:bdw */
7305         I915_WRITE(GEN7_FF_THREAD_MODE,
7306                    I915_READ(GEN7_FF_THREAD_MODE) &
7307                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7308
7309         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7310                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7311
7312         /* WaDisableSDEUnitClockGating:bdw */
7313         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7314                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7315
7316         /* WaProgramL3SqcReg1Default:bdw */
7317         gen8_set_l3sqc_credits(dev_priv, 30, 2);
7318
7319         /* WaKVMNotificationOnConfigChange:bdw */
7320         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7321                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7322
7323         lpt_init_clock_gating(dev_priv);
7324
7325         /* WaDisableDopClockGating:bdw
7326          *
7327          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7328          * clock gating.
7329          */
7330         I915_WRITE(GEN6_UCGCTL1,
7331                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7332 }
7333
7334 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7335 {
7336         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7337         I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7338                    I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7339                    HSW_FBCQ_DIS);
7340
7341         /* This is required by WaCatErrorRejectionIssue:hsw */
7342         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7343                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7344                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7345
7346         /* WaSwitchSolVfFArbitrationPriority:hsw */
7347         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7348
7349         lpt_init_clock_gating(dev_priv);
7350 }
7351
7352 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7353 {
7354         u32 snpcr;
7355
7356         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7357
7358         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7359         I915_WRITE(ILK_DISPLAY_CHICKEN1,
7360                    I915_READ(ILK_DISPLAY_CHICKEN1) |
7361                    ILK_FBCQ_DIS);
7362
7363         /* WaDisableBackToBackFlipFix:ivb */
7364         I915_WRITE(IVB_CHICKEN3,
7365                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7366                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7367
7368         if (IS_IVB_GT1(dev_priv))
7369                 I915_WRITE(GEN7_ROW_CHICKEN2,
7370                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7371         else {
7372                 /* must write both registers */
7373                 I915_WRITE(GEN7_ROW_CHICKEN2,
7374                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7375                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7376                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7377         }
7378
7379         /*
7380          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7381          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7382          */
7383         I915_WRITE(GEN6_UCGCTL2,
7384                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7385
7386         /* This is required by WaCatErrorRejectionIssue:ivb */
7387         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7388                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7389                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7390
7391         g4x_disable_trickle_feed(dev_priv);
7392
7393         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7394         snpcr &= ~GEN6_MBC_SNPCR_MASK;
7395         snpcr |= GEN6_MBC_SNPCR_MED;
7396         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7397
7398         if (!HAS_PCH_NOP(dev_priv))
7399                 cpt_init_clock_gating(dev_priv);
7400
7401         gen6_check_mch_setup(dev_priv);
7402 }
7403
7404 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7405 {
7406         /* WaDisableBackToBackFlipFix:vlv */
7407         I915_WRITE(IVB_CHICKEN3,
7408                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7409                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7410
7411         /* WaDisableDopClockGating:vlv */
7412         I915_WRITE(GEN7_ROW_CHICKEN2,
7413                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7414
7415         /* This is required by WaCatErrorRejectionIssue:vlv */
7416         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7417                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7418                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7419
7420         /*
7421          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7422          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7423          */
7424         I915_WRITE(GEN6_UCGCTL2,
7425                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7426
7427         /* WaDisableL3Bank2xClockGate:vlv
7428          * Disabling L3 clock gating- MMIO 940c[25] = 1
7429          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7430         I915_WRITE(GEN7_UCGCTL4,
7431                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7432
7433         /*
7434          * WaDisableVLVClockGating_VBIIssue:vlv
7435          * Disable clock gating on th GCFG unit to prevent a delay
7436          * in the reporting of vblank events.
7437          */
7438         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7439 }
7440
7441 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7442 {
7443         /* WaVSRefCountFullforceMissDisable:chv */
7444         /* WaDSRefCountFullforceMissDisable:chv */
7445         I915_WRITE(GEN7_FF_THREAD_MODE,
7446                    I915_READ(GEN7_FF_THREAD_MODE) &
7447                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7448
7449         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7450         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7451                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7452
7453         /* WaDisableCSUnitClockGating:chv */
7454         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7455                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7456
7457         /* WaDisableSDEUnitClockGating:chv */
7458         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7459                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7460
7461         /*
7462          * WaProgramL3SqcReg1Default:chv
7463          * See gfxspecs/Related Documents/Performance Guide/
7464          * LSQC Setting Recommendations.
7465          */
7466         gen8_set_l3sqc_credits(dev_priv, 38, 2);
7467 }
7468
7469 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7470 {
7471         u32 dspclk_gate;
7472
7473         I915_WRITE(RENCLK_GATE_D1, 0);
7474         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7475                    GS_UNIT_CLOCK_GATE_DISABLE |
7476                    CL_UNIT_CLOCK_GATE_DISABLE);
7477         I915_WRITE(RAMCLK_GATE_D, 0);
7478         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7479                 OVRUNIT_CLOCK_GATE_DISABLE |
7480                 OVCUNIT_CLOCK_GATE_DISABLE;
7481         if (IS_GM45(dev_priv))
7482                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7483         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7484
7485         g4x_disable_trickle_feed(dev_priv);
7486 }
7487
7488 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7489 {
7490         struct intel_uncore *uncore = &dev_priv->uncore;
7491
7492         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7493         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7494         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7495         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7496         intel_uncore_write16(uncore, DEUC, 0);
7497         intel_uncore_write(uncore,
7498                            MI_ARB_STATE,
7499                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7500 }
7501
7502 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7503 {
7504         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7505                    I965_RCC_CLOCK_GATE_DISABLE |
7506                    I965_RCPB_CLOCK_GATE_DISABLE |
7507                    I965_ISC_CLOCK_GATE_DISABLE |
7508                    I965_FBC_CLOCK_GATE_DISABLE);
7509         I915_WRITE(RENCLK_GATE_D2, 0);
7510         I915_WRITE(MI_ARB_STATE,
7511                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7512 }
7513
7514 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7515 {
7516         u32 dstate = I915_READ(D_STATE);
7517
7518         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7519                 DSTATE_DOT_CLOCK_GATING;
7520         I915_WRITE(D_STATE, dstate);
7521
7522         if (IS_PINEVIEW(dev_priv))
7523                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7524
7525         /* IIR "flip pending" means done if this bit is set */
7526         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7527
7528         /* interrupts should cause a wake up from C3 */
7529         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7530
7531         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7532         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7533
7534         I915_WRITE(MI_ARB_STATE,
7535                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7536 }
7537
7538 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7539 {
7540         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7541
7542         /* interrupts should cause a wake up from C3 */
7543         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7544                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7545
7546         I915_WRITE(MEM_MODE,
7547                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7548
7549         /*
7550          * Have FBC ignore 3D activity since we use software
7551          * render tracking, and otherwise a pure 3D workload
7552          * (even if it just renders a single frame and then does
7553          * abosultely nothing) would not allow FBC to recompress
7554          * until a 2D blit occurs.
7555          */
7556         I915_WRITE(SCPD0,
7557                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7558 }
7559
7560 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7561 {
7562         I915_WRITE(MEM_MODE,
7563                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7564                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7565 }
7566
7567 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7568 {
7569         dev_priv->display.init_clock_gating(dev_priv);
7570 }
7571
7572 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7573 {
7574         if (HAS_PCH_LPT(dev_priv))
7575                 lpt_suspend_hw(dev_priv);
7576 }
7577
7578 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7579 {
7580         drm_dbg_kms(&dev_priv->drm,
7581                     "No clock gating settings or workarounds applied.\n");
7582 }
7583
7584 /**
7585  * intel_init_clock_gating_hooks - setup the clock gating hooks
7586  * @dev_priv: device private
7587  *
7588  * Setup the hooks that configure which clocks of a given platform can be
7589  * gated and also apply various GT and display specific workarounds for these
7590  * platforms. Note that some GT specific workarounds are applied separately
7591  * when GPU contexts or batchbuffers start their execution.
7592  */
7593 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7594 {
7595         if (IS_GEN(dev_priv, 12))
7596                 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7597         else if (IS_GEN(dev_priv, 11))
7598                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7599         else if (IS_CANNONLAKE(dev_priv))
7600                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7601         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7602                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7603         else if (IS_SKYLAKE(dev_priv))
7604                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7605         else if (IS_KABYLAKE(dev_priv))
7606                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7607         else if (IS_BROXTON(dev_priv))
7608                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7609         else if (IS_GEMINILAKE(dev_priv))
7610                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7611         else if (IS_BROADWELL(dev_priv))
7612                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7613         else if (IS_CHERRYVIEW(dev_priv))
7614                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7615         else if (IS_HASWELL(dev_priv))
7616                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7617         else if (IS_IVYBRIDGE(dev_priv))
7618                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7619         else if (IS_VALLEYVIEW(dev_priv))
7620                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7621         else if (IS_GEN(dev_priv, 6))
7622                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7623         else if (IS_GEN(dev_priv, 5))
7624                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7625         else if (IS_G4X(dev_priv))
7626                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7627         else if (IS_I965GM(dev_priv))
7628                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7629         else if (IS_I965G(dev_priv))
7630                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7631         else if (IS_GEN(dev_priv, 3))
7632                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7633         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7634                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7635         else if (IS_GEN(dev_priv, 2))
7636                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7637         else {
7638                 MISSING_CASE(INTEL_DEVID(dev_priv));
7639                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7640         }
7641 }
7642
7643 /* Set up chip specific power management-related functions */
7644 void intel_init_pm(struct drm_i915_private *dev_priv)
7645 {
7646         /* For cxsr */
7647         if (IS_PINEVIEW(dev_priv))
7648                 pnv_get_mem_freq(dev_priv);
7649         else if (IS_GEN(dev_priv, 5))
7650                 ilk_get_mem_freq(dev_priv);
7651
7652         if (intel_has_sagv(dev_priv))
7653                 skl_setup_sagv_block_time(dev_priv);
7654
7655         /* For FIFO watermark updates */
7656         if (INTEL_GEN(dev_priv) >= 9) {
7657                 skl_setup_wm_latency(dev_priv);
7658                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7659         } else if (HAS_PCH_SPLIT(dev_priv)) {
7660                 ilk_setup_wm_latency(dev_priv);
7661
7662                 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7663                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7664                     (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7665                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7666                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7667                         dev_priv->display.compute_intermediate_wm =
7668                                 ilk_compute_intermediate_wm;
7669                         dev_priv->display.initial_watermarks =
7670                                 ilk_initial_watermarks;
7671                         dev_priv->display.optimize_watermarks =
7672                                 ilk_optimize_watermarks;
7673                 } else {
7674                         drm_dbg_kms(&dev_priv->drm,
7675                                     "Failed to read display plane latency. "
7676                                     "Disable CxSR\n");
7677                 }
7678         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7679                 vlv_setup_wm_latency(dev_priv);
7680                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7681                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7682                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7683                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7684                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7685         } else if (IS_G4X(dev_priv)) {
7686                 g4x_setup_wm_latency(dev_priv);
7687                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7688                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7689                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7690                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7691         } else if (IS_PINEVIEW(dev_priv)) {
7692                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7693                                             dev_priv->is_ddr3,
7694                                             dev_priv->fsb_freq,
7695                                             dev_priv->mem_freq)) {
7696                         drm_info(&dev_priv->drm,
7697                                  "failed to find known CxSR latency "
7698                                  "(found ddr%s fsb freq %d, mem freq %d), "
7699                                  "disabling CxSR\n",
7700                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7701                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7702                         /* Disable CxSR and never update its watermark again */
7703                         intel_set_memory_cxsr(dev_priv, false);
7704                         dev_priv->display.update_wm = NULL;
7705                 } else
7706                         dev_priv->display.update_wm = pnv_update_wm;
7707         } else if (IS_GEN(dev_priv, 4)) {
7708                 dev_priv->display.update_wm = i965_update_wm;
7709         } else if (IS_GEN(dev_priv, 3)) {
7710                 dev_priv->display.update_wm = i9xx_update_wm;
7711                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7712         } else if (IS_GEN(dev_priv, 2)) {
7713                 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7714                         dev_priv->display.update_wm = i845_update_wm;
7715                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7716                 } else {
7717                         dev_priv->display.update_wm = i9xx_update_wm;
7718                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7719                 }
7720         } else {
7721                 drm_err(&dev_priv->drm,
7722                         "unexpected fall-through in %s\n", __func__);
7723         }
7724 }
7725
7726 void intel_pm_setup(struct drm_i915_private *dev_priv)
7727 {
7728         dev_priv->runtime_pm.suspended = false;
7729         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7730 }
7731
7732 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7733 {
7734         struct intel_dbuf_state *dbuf_state;
7735
7736         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7737         if (!dbuf_state)
7738                 return NULL;
7739
7740         return &dbuf_state->base;
7741 }
7742
7743 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7744                                      struct intel_global_state *state)
7745 {
7746         kfree(state);
7747 }
7748
7749 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7750         .atomic_duplicate_state = intel_dbuf_duplicate_state,
7751         .atomic_destroy_state = intel_dbuf_destroy_state,
7752 };
7753
7754 struct intel_dbuf_state *
7755 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7756 {
7757         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7758         struct intel_global_state *dbuf_state;
7759
7760         dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7761         if (IS_ERR(dbuf_state))
7762                 return ERR_CAST(dbuf_state);
7763
7764         return to_intel_dbuf_state(dbuf_state);
7765 }
7766
7767 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7768 {
7769         struct intel_dbuf_state *dbuf_state;
7770
7771         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7772         if (!dbuf_state)
7773                 return -ENOMEM;
7774
7775         intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7776                                      &dbuf_state->base, &intel_dbuf_funcs);
7777
7778         return 0;
7779 }
7780
7781 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7782 {
7783         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7784         const struct intel_dbuf_state *new_dbuf_state =
7785                 intel_atomic_get_new_dbuf_state(state);
7786         const struct intel_dbuf_state *old_dbuf_state =
7787                 intel_atomic_get_old_dbuf_state(state);
7788
7789         if (!new_dbuf_state ||
7790             new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7791                 return;
7792
7793         WARN_ON(!new_dbuf_state->base.changed);
7794
7795         gen9_dbuf_slices_update(dev_priv,
7796                                 old_dbuf_state->enabled_slices |
7797                                 new_dbuf_state->enabled_slices);
7798 }
7799
7800 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7801 {
7802         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7803         const struct intel_dbuf_state *new_dbuf_state =
7804                 intel_atomic_get_new_dbuf_state(state);
7805         const struct intel_dbuf_state *old_dbuf_state =
7806                 intel_atomic_get_old_dbuf_state(state);
7807
7808         if (!new_dbuf_state ||
7809             new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7810                 return;
7811
7812         WARN_ON(!new_dbuf_state->base.changed);
7813
7814         gen9_dbuf_slices_update(dev_priv,
7815                                 new_dbuf_state->enabled_slices);
7816 }